diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/manifest.txt b/tierb_study_20260826_144644/ultraep/tierbprobe/manifest.txt new file mode 100644 index 0000000000000000000000000000000000000000..8666098ab488ef5a2919e471a36bc86c88bac72e --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/manifest.txt @@ -0,0 +1,19 @@ +timestamp: 2026-08-26T14:58:12+00:00 +host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 +gpu: NVIDIA B200 x8 +driver: 580.95.05 +study_scripts_branch: granularity-ultraep +study_scripts_commit: 81efb04 +campaign: ultraep +backend: ultraep +phase: tierbprobe +gbs: 8 +seq: 4096 +configs: G1 G2 G3 G4 +capacity: 1.25 +repeats: 3 +nsys: 1 +megatron_repo: /workspace/Megatron-UltraEP +megatron_branch_required: ultraep-token-drop-analysis +megatron_branch_actual: ultraep-token-drop-analysis +megatron_commit: eaf829e7 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/pip_freeze.txt b/tierb_study_20260826_144644/ultraep/tierbprobe/pip_freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..7b269e48e7b27ebf2a36d76baa683b15661c1a26 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/pip_freeze.txt @@ -0,0 +1,317 @@ +absl-py==2.3.0 +aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10 +aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5 +annotated-types==0.7.0 +anyio==4.9.0 +apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170 +argon2-cffi==25.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4 +asttokens==3.0.0 +astunparse==1.6.3 +async-lru==2.0.5 +attrs==25.3.0 +audioread==3.0.1 +babel==2.17.0 +beautifulsoup4==4.13.4 +black==25.1.0 +bleach==6.2.0 +blis==0.7.11 +cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e +catalogue==2.0.10 +certifi==2025.4.26 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.5.0 +cloudpathlib==0.21.1 +cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e +cmake==3.31.6 +comm==0.2.2 +confection==0.1.5 +contourpy==1.3.2 +cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e +cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7 +cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348 +cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869 +cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29 +cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470 +cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be +cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5 +cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f +cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914 +cycler==0.12.1 +cymem==2.0.11 +Cython==3.1.1 +dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e +dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19 +dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c +datasets==5.0.1 +debugpy==1.8.14 +decorator==5.2.1 +-e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep +defusedxml==0.7.1 +dill==0.4.1 +distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67 +distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d +dm-tree==0.1.9 +einops==0.8.1 +execnet==2.1.1 +executing==2.2.0 +expecttest==0.3.0 +fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 +fastjsonschema==2.21.1 +fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670 +filelock==3.18.0 +flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a +fonttools==4.58.1 +fqdn==1.5.1 +frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103 +fsspec==2025.5.1 +gast==0.6.0 +grpcio==1.62.1 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.28.0 +hypothesis==6.130.8 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +intel-openmp==2021.4.0 +ipykernel==6.29.5 +ipython==9.3.0 +ipython_pygments_lexers==1.1.1 +isoduration==20.11.0 +isort==6.0.1 +jedi==0.19.2 +Jinja2==3.1.6 +joblib==1.5.1 +json5==0.12.0 +jsonpointer==3.0.0 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +jupyter-events==0.12.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.3 +jupyter_core==5.8.1 +jupyter_server==2.16.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.4.3 +jupyterlab_code_formatter==3.0.2 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e +jupytext==1.17.2 +kiwisolver==1.4.8 +kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85 +langcodes==3.5.0 +language_data==1.3.0 +lazy_loader==0.4 +libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4 +libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d +libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c +libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9 +libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7 +libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24 +librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f +librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3 +librosa==0.11.0 +libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566 +lightning-thunder @ file:///opt/pytorch/lightning-thunder +lightning-utilities==0.14.3 +lintrunner==0.12.7 +llvmlite==0.42.0 +locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3 +looseversion==1.3.0 +marisa-trie==1.2.1 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.3 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.4.2 +mdurl==0.1.2 +-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@b099fd8494b09e8c08ad34e26abf91fb6c83636d#egg=megatron_core +mistune==3.1.3 +mkl==2021.1.1 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +ml_dtypes==0.6.0 +mock==5.2.0 +-e git+ssh://git@github.com/sriram-17-17/MoonEP.git@c27d35da15f25c112218795894504a65c16f4879#egg=moonep +mpmath==1.3.0 +msgpack==1.1.0 +multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1 +multiprocess==0.70.19 +murmurhash==1.0.13 +mypy_extensions==1.1.0 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.5 +ninja==1.11.1.4 +notebook==7.4.3 +notebook_shim==0.2.4 +numba==0.59.1 +numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a +numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53 +numpy==1.26.4 +nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87 +nvfuser==0.2.27a0+9bf5aca +nvidia-cuda-cccl-cu12==12.9.27 +nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend +nvidia-cutlass-dsl==4.4.2 +nvidia-cutlass-dsl-libs-base==4.4.2 +nvidia-dali-cuda120==1.50.0 +nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e +nvidia-modelopt==0.29.0 +nvidia-modelopt-core==0.29.0 +nvidia-nvcomp-cu12==4.2.0.14 +nvidia-nvimgcodec-cu12==0.5.0.13 +nvidia-nvjpeg-cu12==12.4.0.16 +nvidia-nvjpeg2k-cu12==0.8.1.40 +nvidia-nvshmem-cu12==3.7.2 +nvidia-nvtiff-cu12==0.5.0.67 +nvidia-resiliency-ext==0.4.0 +nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e +nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111 +onnx @ file:///opt/pytorch/pytorch/third_party/onnx +onnx-ir==1.0.0 +onnxscript==0.7.1 +opt_einsum==3.4.0 +optree==0.16.0 +overrides==7.7.0 +packaging==23.2 +pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319 +pandocfilters==1.5.1 +parso==0.8.4 +partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f +pathspec==0.12.1 +pexpect==4.9.0 +pillow==11.2.1 +platformdirs==4.3.8 +pluggy==1.6.0 +ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce +polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09 +polygraphy==0.49.20 +pooch==1.8.2 +preshed==3.0.10 +prometheus_client==0.22.1 +prompt_toolkit==3.0.51 +propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25 +protobuf==4.24.4 +psutil==7.0.0 +ptyprocess==0.7.0 +PuLP==3.2.1 +pure_eval==0.2.3 +pyarrow==25.0.1 +pybind11==2.13.6 +pybind11_global==2.13.6 +pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI +pycparser==2.22 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.19.1 +pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a +pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557 +pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a +pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e +pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2 +pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919 +pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e +pyparsing==3.2.3 +pytest==8.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==15.1 +pytest-shard==0.1.2 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-hostlist==2.2.1 +python-json-logger==3.3.0 +pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d +pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a +PyYAML==6.0.2 +pyzmq==26.4.0 +raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c +rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd +rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==14.0.0 +rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712 +rpds-py==0.25.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.15.3 +Send2Trash==1.8.3 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.16.0 +smart-open==7.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.13.1 +soupsieve==2.7 +soxr==0.5.0.post1 +spacy==3.7.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +srsly==2.5.1 +stack-data==0.6.3 +sympy==1.14.0 +tabulate==0.9.0 +tbb==2021.13.1 +tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a +terminado==0.18.1 +thinc==8.2.5 +threadpoolctl==3.6.0 +thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147 +tinycss2==1.4.0 +toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236 +torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5 +torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1 +torchao @ file:///opt/pytorch/ao +torchprofile==0.0.4 +torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d +tornado==6.5.1 +tqdm==4.67.1 +traitlets==5.14.3 +transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189 +treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb +typer==0.16.0 +types-dataclasses==0.6.6 +types-python-dateutil==2.9.0.20250516 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 +ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a +ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f +-e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep +uri-template==1.3.0 +urllib3==2.0.7 +wasabi==1.1.3 +wcwidth==0.2.13 +weasel==0.4.1 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +xdoctest==1.0.2 +xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b +xxhash==4.0.1 +yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018 +zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 +zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae +zipp==3.22.0 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/provenance.txt b/tierb_study_20260826_144644/ultraep/tierbprobe/provenance.txt new file mode 100644 index 0000000000000000000000000000000000000000..4b7329475d5c7131fa1c71f5f3adefa0305d4d42 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/provenance.txt @@ -0,0 +1,90 @@ +============================================================== +PROVENANCE: ultraep__tierbprobe__gbs8 +============================================================== +captured_at : 2026-08-26T14:57:57+00:00 +hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 +kernel : 6.8.0-1043-nvidia +cwd : /workspace/blackwell_run_scripts +command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_study_20260826_144644/ultraep/tierbprobe --nsys + +---- GPU / driver -------------------------------------------- + 0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + persistence/clocks: + 0, 120 MHz, 1965 MHz, 1000.00 W, 29 + 1, 120 MHz, 1965 MHz, 1000.00 W, 34 + 2, 120 MHz, 1965 MHz, 1000.00 W, 34 + 3, 120 MHz, 1965 MHz, 1000.00 W, 30 + 4, 120 MHz, 1965 MHz, 1000.00 W, 29 + 5, 120 MHz, 1965 MHz, 1000.00 W, 34 + 6, 120 MHz, 1965 MHz, 1000.00 W, 34 + 7, 120 MHz, 1965 MHz, 1000.00 W, 29 + + NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.28.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo token-drop-analysis b099fd8494b0 + Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21 + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-integration c27d35da15f2 + blackwell_run_scripts granularity-ultraep 81efb04918d8 + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=tierb_study_20260826_144644 + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Aug 26 14:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Aug 26 14:09 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..c0143e953e07278a6e327d16fdf12af5468e873a --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1/mem_samples.csv @@ -0,0 +1,152 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 27 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 933 +1, 897 +2, 919 +3, 909 +4, 903 +5, 893 +6, 919 +7, 927 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 2473 +1, 2475 +2, 2481 +3, 2481 +4, 2473 +5, 2465 +6, 2481 +7, 2481 +0, 25239 +1, 16131 +2, 25239 +3, 25239 +4, 16131 +5, 25239 +6, 16131 +7, 25239 +0, 28867 +1, 28977 +2, 28835 +3, 28859 +4, 28757 +5, 28827 +6, 28929 +7, 28967 +0, 39035 +1, 39139 +2, 38965 +3, 38989 +4, 38951 +5, 38939 +6, 39083 +7, 39119 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 38825 +1, 38909 +2, 38759 +3, 38759 +4, 38759 +5, 38755 +6, 38891 +7, 38909 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..cf481ddf67e28b1a873dc310db0ffef890c69b4a --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.02 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=768 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..48fc91caf8f8912bc5ea931fda3aac73e4d17f00 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1/train.log @@ -0,0 +1,2349 @@ +[15:03:40] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + DSV2_LITE_L24 / ultraep (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.02 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 15:03:41.582000 58465 torch/distributed/run.py:766] +W0826 15:03:41.582000 58465 torch/distributed/run.py:766] ***************************************** +W0826 15:03:41.582000 58465 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:03:41.582000 58465 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.02 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 2[Gloo] Rank 346 is connected to is connected to 5 is connected to is connected to 77 is connected to 77 peer ranks. peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77Expected number of connected peer ranks is : +77 +7 + + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 76 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to [Gloo] Rank is connected to 000 peer ranks. is connected to peer ranks. Expected number of connected peer ranks is : 0Expected number of connected peer ranks is : 0 peer ranks. 0 +Expected number of connected peer ranks is : +0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W826 15:03:59.862015990 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:03:59.867558408 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:03:59.871487236 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:03:59.871559849 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:03:59.871609901 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:03:59.872652645 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:03:59.872692163 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 15:03:59.023314363 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.434 seconds +time to initialize megatron (seconds): 6.506 +[after megatron is initialized] datetime: 2026-08-26 15:04:05 +building GPT model ... +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4188125184 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.input_layernorm.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:04:09 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 15:04:09 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3661.88, 3668.77) + train/valid/test-data-iterators-setup ..........: (68.04, 86.13) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 15:04:09 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.utils:num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +[rank7]: Traceback (most recent call last): +[rank7]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank7]: pretrain( +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank7]: iteration, num_floating_point_operations_so_far = train( +[rank7]: ^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank7]: ) = train_step( +[rank7]: ^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank7]: losses_reduced = forward_backward_func( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank7]: output_tensor, num_tokens = forward_step( +[rank7]: ^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank7]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank7]: output_tensor = model( +[rank7]: ^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank7]: return self.module(*inputs, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank7]: outputs = self.module(*inputs, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank7]: hidden_states = self.decoder( +[rank7]: ^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank7]: hidden_states, context = layer( +[rank7]: ^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank7]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank7]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank7]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank7]: return self.ultraep_forward(hidden_states) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank7]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank7]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank7]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank7]: hidden_states = self._comm_manager.dispatch( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank7]: ) = hybrid_ep_dispatch( +[rank7]: ^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank7]: return HybridEPDispatch.apply( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank7]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank7]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank7]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank2]: Traceback (most recent call last): +[rank2]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank2]: pretrain( +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank2]: iteration, num_floating_point_operations_so_far = train( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank2]: ) = train_step( +[rank2]: ^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank2]: losses_reduced = forward_backward_func( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank2]: output_tensor, num_tokens = forward_step( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank2]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank2]: output_tensor = model( +[rank2]: ^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank2]: return self.module(*inputs, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank2]: outputs = self.module(*inputs, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank2]: hidden_states = self.decoder( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank2]: hidden_states, context = layer( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank2]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank2]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank2]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank2]: return self.ultraep_forward(hidden_states) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank2]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank2]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank2]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank2]: hidden_states = self._comm_manager.dispatch( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank2]: ) = hybrid_ep_dispatch( +[rank2]: ^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank2]: return HybridEPDispatch.apply( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank2]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank2]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank2]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank5]: Traceback (most recent call last): +[rank5]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank5]: pretrain( +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank5]: iteration, num_floating_point_operations_so_far = train( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank5]: ) = train_step( +[rank5]: ^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank5]: losses_reduced = forward_backward_func( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank5]: output_tensor, num_tokens = forward_step( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank5]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank5]: output_tensor = model( +[rank5]: ^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank5]: return self.module(*inputs, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank5]: outputs = self.module(*inputs, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank5]: hidden_states = self.decoder( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank5]: hidden_states, context = layer( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank5]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank5]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank5]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank5]: return self.ultraep_forward(hidden_states) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank5]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank5]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank5]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank5]: hidden_states = self._comm_manager.dispatch( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank5]: ) = hybrid_ep_dispatch( +[rank5]: ^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank5]: return HybridEPDispatch.apply( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank5]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank5]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank5]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank6]: Traceback (most recent call last): +[rank6]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank6]: pretrain( +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank6]: iteration, num_floating_point_operations_so_far = train( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank6]: ) = train_step( +[rank6]: ^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank6]: losses_reduced = forward_backward_func( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank6]: output_tensor, num_tokens = forward_step( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank6]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank6]: output_tensor = model( +[rank6]: ^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank6]: return self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank6]: outputs = self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank6]: hidden_states = self.decoder( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank6]: hidden_states, context = layer( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank6]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank6]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank6]: return self.ultraep_forward(hidden_states) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank6]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank6]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank6]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank6]: hidden_states = self._comm_manager.dispatch( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank6]: ) = hybrid_ep_dispatch( +[rank6]: ^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank6]: return HybridEPDispatch.apply( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank6]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank6]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank1]: Traceback (most recent call last): +[rank1]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank1]: pretrain( +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank1]: ) = train_step( +[rank1]: ^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank1]: output_tensor = model( +[rank1]: ^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank1]: hidden_states = self.decoder( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank1]: hidden_states, context = layer( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank1]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank1]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank1]: return self.ultraep_forward(hidden_states) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank1]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank1]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank1]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank1]: hidden_states = self._comm_manager.dispatch( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank1]: ) = hybrid_ep_dispatch( +[rank1]: ^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank1]: return HybridEPDispatch.apply( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank1]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank1]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank4]: Traceback (most recent call last): +[rank4]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank4]: pretrain( +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank4]: iteration, num_floating_point_operations_so_far = train( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank4]: ) = train_step( +[rank4]: ^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank4]: losses_reduced = forward_backward_func( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank4]: output_tensor, num_tokens = forward_step( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank4]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank4]: output_tensor = model( +[rank4]: ^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank4]: return self.module(*inputs, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank4]: outputs = self.module(*inputs, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank4]: hidden_states = self.decoder( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank4]: hidden_states, context = layer( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank4]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank4]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank4]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank4]: return self.ultraep_forward(hidden_states) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank4]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank4]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank4]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank4]: hidden_states = self._comm_manager.dispatch( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank4]: ) = hybrid_ep_dispatch( +[rank4]: ^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank4]: return HybridEPDispatch.apply( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank4]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank4]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank4]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank0]: Traceback (most recent call last): +[rank0]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank0]: pretrain( +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: ^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank0]: ) = train_step( +[rank0]: ^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank0]: output_tensor = model( +[rank0]: ^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank0]: hidden_states = self.decoder( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank0]: hidden_states, context = layer( +[rank0]: ^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank0]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank0]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank0]: return self.ultraep_forward(hidden_states) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank0]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank0]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank0]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank0]: hidden_states = self._comm_manager.dispatch( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank0]: ) = hybrid_ep_dispatch( +[rank0]: ^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank0]: return HybridEPDispatch.apply( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank0]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank0]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank3]: Traceback (most recent call last): +[rank3]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank3]: pretrain( +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank3]: iteration, num_floating_point_operations_so_far = train( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank3]: ) = train_step( +[rank3]: ^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank3]: losses_reduced = forward_backward_func( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank3]: output_tensor, num_tokens = forward_step( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank3]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank3]: output_tensor = model( +[rank3]: ^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank3]: return self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank3]: outputs = self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank3]: hidden_states = self.decoder( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank3]: hidden_states, context = layer( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank3]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank3]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank3]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank3]: return self.ultraep_forward(hidden_states) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank3]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank3]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank3]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank3]: hidden_states = self._comm_manager.dispatch( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank3]: ) = hybrid_ep_dispatch( +[rank3]: ^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank3]: return HybridEPDispatch.apply( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank3]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank3]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank3]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: AssertionError: num_permuted_tokens (50135) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank0]:[W826 15:04:25.262373764 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:04:26.425000 58465 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 58555 closing signal SIGTERM +W0826 15:04:26.426000 58465 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 58556 closing signal SIGTERM +W0826 15:04:26.426000 58465 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 58557 closing signal SIGTERM +W0826 15:04:26.427000 58465 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 58558 closing signal SIGTERM +W0826 15:04:26.427000 58465 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 58559 closing signal SIGTERM +W0826 15:04:26.427000 58465 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 58561 closing signal SIGTERM +W0826 15:04:26.428000 58465 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 58562 closing signal SIGTERM +E0826 15:04:28.973000 58465 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 5 (pid: 58560) of binary: /usr/bin/python +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-08-26_15:04:26 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 5 (local_rank: 5) + exitcode : 1 (pid: 58560) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..0812871d3f04ec69febfb8f42e66394d3be53ff6 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 761 +1, 725 +2, 737 +3, 725 +4, 725 +5, 725 +6, 747 +7, 725 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2471 +1, 2471 +2, 2471 +3, 2471 +4, 2471 +5, 2471 +6, 2471 +7, 2471 +0, 2871 +1, 2871 +2, 2871 +3, 2871 +4, 2871 +5, 2871 +6, 2871 +7, 2871 +0, 28477 +1, 28565 +2, 28411 +3, 28411 +4, 28411 +5, 28411 +6, 28543 +7, 28565 +0, 34335 +1, 34419 +2, 34269 +3, 34267 +4, 34267 +5, 34265 +6, 34401 +7, 34419 +0, 39471 +1, 39555 +2, 39405 +3, 39405 +4, 39405 +5, 39401 +6, 39537 +7, 39555 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 39779 +1, 39863 +2, 39713 +3, 39713 +4, 39713 +5, 39709 +6, 39845 +7, 39863 +0, 39781 +1, 39865 +2, 39715 +3, 39715 +4, 39715 +5, 39711 +6, 39847 +7, 39865 +0, 111665 +1, 111781 +2, 111693 +3, 111607 +4, 111671 +5, 111589 +6, 111749 +7, 111745 +0, 112149 +1, 112235 +2, 112149 +3, 112083 +4, 112147 +5, 112081 +6, 112215 +7, 112235 +0, 112155 +1, 112239 +2, 112153 +3, 112089 +4, 112153 +5, 112085 +6, 112221 +7, 112239 +0, 112155 +1, 112239 +2, 112153 +3, 112089 +4, 112153 +5, 112085 +6, 112221 +7, 112239 +0, 114517 +1, 112583 +2, 114539 +3, 114495 +4, 114525 +5, 114443 +6, 112557 +7, 112611 +0, 115425 +1, 113483 +2, 115419 +3, 115389 +4, 115449 +5, 115383 +6, 113435 +7, 113495 +0, 127845 +1, 129971 +2, 127845 +3, 129825 +4, 129889 +5, 129817 +6, 130023 +7, 129971 +0, 127845 +1, 129971 +2, 127845 +3, 129825 +4, 129889 +5, 129817 +6, 130023 +7, 129971 +0, 127845 +1, 129971 +2, 127845 +3, 129825 +4, 129889 +5, 129817 +6, 130023 +7, 129971 +0, 127845 +1, 129971 +2, 127845 +3, 129825 +4, 129889 +5, 129817 +6, 130023 +7, 129971 +0, 126899 +1, 129025 +2, 126899 +3, 128879 +4, 128943 +5, 128871 +6, 129077 +7, 129025 +0, 124071 +1, 125963 +2, 124071 +3, 126051 +4, 125727 +5, 125723 +6, 126089 +7, 126197 +0, 321 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 68 +7, 124723 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..43b070e90f2501791de7a6b5b1d8cd8c5fcb819c --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=768 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..830b08367c3622c74934514134049aa0196af626 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log @@ -0,0 +1,12749 @@ +[15:02:25] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + DSV2_LITE_L24 / ultraep (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 15:02:27.046000 54788 torch/distributed/run.py:766] +W0826 15:02:27.046000 54788 torch/distributed/run.py:766] ***************************************** +W0826 15:02:27.046000 54788 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:02:27.046000 54788 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 23 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 203 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 + + +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to [Gloo] Rank 7 peer ranks. 7Expected number of connected peer ranks is : is connected to 77 + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to [Gloo] Rank 0 peer ranks. 0Expected number of connected peer ranks is : is connected to 00 + peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W826 15:02:45.186808861 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:02:45.186966330 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:02:45.189668353 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:02:45.189719675 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:02:45.189816631 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:02:45.189828922 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:02:45.191376132 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 15:02:45.355922139 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.382 seconds +time to initialize megatron (seconds): 6.708 +[after megatron is initialized] datetime: 2026-08-26 15:02:50 +building GPT model ... +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4188125184 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.input_layernorm.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:02:54 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 15:02:54 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3598.16, 3604.40) + train/valid/test-data-iterators-setup ..........: (71.76, 87.61) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 15:02:54 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48667 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49446 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=47653 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48873 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48667 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49446 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48420 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=47653 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48420 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48558 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48558 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48121 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48121 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48018 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48018 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49529 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49529 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48519 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48519 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=47476 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=47476 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=61440 expected_recv=46521 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49469 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49544 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=61440 expected_recv=46521 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49469 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49544 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49294 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=47251 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49294 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=47251 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=61440 expected_recv=47103 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=61440 expected_recv=47103 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=61440 expected_recv=47521 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=61440 expected_recv=47521 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=47831 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48164 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=47831 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48164 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48450 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=61440 expected_recv=47882 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48450 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48382 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=61440 expected_recv=47882 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48382 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=47127 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48668 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=47127 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48668 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=61440 expected_recv=46684 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=61440 expected_recv=46684 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=61440 expected_recv=47266 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48085 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=61440 expected_recv=47266 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48085 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48551 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48551 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49373 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49373 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=45979 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=48765 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=45979 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=48765 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48787 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48787 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=46872 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=46872 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47986 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47986 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49233 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49233 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=47552 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=47552 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48362 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48362 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48225 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48225 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48392 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48392 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=47805 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=47805 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49489 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49489 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47735 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47735 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48645 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48645 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49118 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49118 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=47235 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=46883 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48469 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=47235 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=46883 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48469 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=61440 expected_recv=47459 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=61440 expected_recv=47459 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47777 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47777 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=46765 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48587 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=46765 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48587 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48910 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48353 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48353 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47617 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47617 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=61440 expected_recv=47894 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=61440 expected_recv=47894 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=61440 expected_recv=47734 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=61440 expected_recv=47734 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=23 total_expected_recv=1133511 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267520 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=23 total_expected_recv=1134190 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265600 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=23 total_expected_recv=1125260 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273152 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=23 total_expected_recv=1122397 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=277120 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=23 total_expected_recv=1132282 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268416 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=23 total_expected_recv=1133313 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267264 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=23 total_expected_recv=1134581 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265472 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=23 total_expected_recv=1128434 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:03:23] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 29181.5 | throughput per GPU (TFLOP/s/GPU): 4.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092127E+01 | loss scale: 1.0 | grad norm: 30.623 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +Number of parameters in transformer block in billions: 13.53 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 37110.0849609375 | max allocated: 104505.06396484375 | reserved: 107726.0 | max reserved: 107726.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=61440 expected_recv=48091 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=61440 expected_recv=47725 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=61440 expected_recv=48091 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=61440 expected_recv=47725 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=61440 expected_recv=47820 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=47977 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=61440 expected_recv=47820 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=47977 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49395 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48763 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49395 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48763 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=47761 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=47761 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48451 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48451 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48529 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48697 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48965 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48529 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48697 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48965 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48708 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=47048 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48708 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=47048 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=47292 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=47292 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48060 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48060 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49224 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49224 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=47967 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=61440 expected_recv=47804 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=47967 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=61440 expected_recv=47804 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48522 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48522 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48762 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48762 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=61440 expected_recv=47712 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49504 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47860 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=61440 expected_recv=47712 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49504 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47860 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47632 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47632 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49530 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49530 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47615 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47615 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47850 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47850 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=47005 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=47005 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=48346 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48877 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48200 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=48346 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48877 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48200 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48500 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48500 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48055 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=61440 expected_recv=47920 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48055 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=61440 expected_recv=47920 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48456 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=46896 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48456 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=46896 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=47125 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=47125 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=61440 expected_recv=46948 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48405 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=61440 expected_recv=46948 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48405 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49325 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49325 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=61440 expected_recv=47820 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=61440 expected_recv=47820 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=47851 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=47116 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=47851 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=47116 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49112 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49112 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48769 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48851 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48769 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48851 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=46024 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=46719 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=46024 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=46719 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=47751 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=47751 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=61440 expected_recv=47601 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=61440 expected_recv=47601 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=61440 expected_recv=46982 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=61440 expected_recv=46982 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49497 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48369 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49497 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47938 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48369 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47561 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47938 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47561 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=23 total_expected_recv=1128437 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=23 total_expected_recv=1136387 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263808 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=23 total_expected_recv=1129195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=23 total_expected_recv=1123546 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=276096 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=23 total_expected_recv=1134273 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=266624 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=23 total_expected_recv=1128391 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=23 total_expected_recv=1134221 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267008 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=23 total_expected_recv=1129518 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:03:24] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 642.7 | throughput per GPU (TFLOP/s/GPU): 198.5 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091995E+01 | loss scale: 1.0 | grad norm: 30.567 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48263 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=61440 expected_recv=47905 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48263 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=61440 expected_recv=47905 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=61440 expected_recv=47135 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=61440 expected_recv=47135 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=48217 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=48217 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48686 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48686 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48651 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=61440 expected_recv=47656 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48651 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=61440 expected_recv=47656 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=61440 expected_recv=47875 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48531 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=61440 expected_recv=47875 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48589 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48531 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48589 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47592 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47592 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47878 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47878 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49526 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49526 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47867 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47484 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47867 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47484 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48019 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=61440 expected_recv=47333 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48019 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=61440 expected_recv=47333 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=47049 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=47049 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=61440 expected_recv=48303 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49472 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=61440 expected_recv=48303 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49472 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48201 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48201 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=47323 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=47323 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=47110 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=47110 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48241 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48241 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=46290 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=46290 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=46959 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=46959 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48392 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48392 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49580 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49580 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=46562 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49639 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=46562 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49639 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48855 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=45713 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48362 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=45713 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48362 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48756 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48756 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=46991 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=46991 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=61440 expected_recv=46596 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=61440 expected_recv=46596 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47358 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47358 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47993 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47993 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=23 total_expected_recv=1135446 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264320 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=23 total_expected_recv=1132664 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265984 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=23 total_expected_recv=1123780 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275200 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=23 total_expected_recv=1130427 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269568 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=23 total_expected_recv=1134893 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263680 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=23 total_expected_recv=1128698 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270080 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=23 total_expected_recv=1126154 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273152 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=23 total_expected_recv=1131906 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267136 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:03:24] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 636.8 | throughput per GPU (TFLOP/s/GPU): 200.4 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952354E+00 | loss scale: 1.0 | grad norm: 42.232 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=47005 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49215 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48777 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=47005 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49215 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48777 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=61440 expected_recv=46897 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=61440 expected_recv=46897 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47626 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47725 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47626 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47725 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=45832 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=45832 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49521 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49521 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=61440 expected_recv=46144 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=61440 expected_recv=46144 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48110 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48110 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=61440 expected_recv=47241 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=61440 expected_recv=47241 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=61440 expected_recv=47562 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=61440 expected_recv=47562 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48153 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48153 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49280 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48176 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48176 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=47177 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=47177 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=46417 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=46417 actual_valid_recv=46976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46976 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=61440 expected_recv=47588 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=61440 expected_recv=47588 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48142 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48142 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=45952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=45952 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48465 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48465 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=46885 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=46885 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=23 total_expected_recv=1131921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268800 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=23 total_expected_recv=1129377 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=23 total_expected_recv=1132497 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269952 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=23 total_expected_recv=1133045 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268416 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=23 total_expected_recv=1131525 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270336 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=23 total_expected_recv=1120395 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=280832 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=23 total_expected_recv=1127327 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275712 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=23 total_expected_recv=1137881 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263936 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 15:03:25] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 640.1 | throughput per GPU (TFLOP/s/GPU): 199.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.477323E+00 | loss scale: 1.0 | grad norm: 7.143 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49498 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=47069 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49498 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48429 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=47069 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48429 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=47721 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=61440 expected_recv=47631 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=47721 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=61440 expected_recv=47631 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48890 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48890 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=61440 expected_recv=46461 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=61440 expected_recv=46461 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48530 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48530 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=61440 expected_recv=47269 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=61440 expected_recv=47269 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=61440 expected_recv=46993 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=61440 expected_recv=46993 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48359 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48359 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=47616 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=61440 expected_recv=47738 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=47616 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=61440 expected_recv=47738 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49536 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49536 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=45818 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=61440 expected_recv=47035 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=45818 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=61440 expected_recv=47035 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48316 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48316 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=61440 expected_recv=47367 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=47985 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=61440 expected_recv=47367 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=47985 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=46554 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=46554 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48798 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48798 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47428 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47428 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47925 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47925 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=61440 expected_recv=48377 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=61440 expected_recv=48377 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=61440 expected_recv=47343 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49277 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=61440 expected_recv=47343 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49277 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=46055 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=46055 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=46713 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=46713 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=61440 expected_recv=48640 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=61440 expected_recv=48640 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48442 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48442 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=61440 expected_recv=46911 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=61440 expected_recv=46911 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=23 total_expected_recv=1139534 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=260864 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=23 total_expected_recv=1122666 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=276480 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=23 total_expected_recv=1136609 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=262784 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=23 total_expected_recv=1123786 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275328 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=23 total_expected_recv=1122043 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=278272 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=23 total_expected_recv=1127280 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272384 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=23 total_expected_recv=1140820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=259584 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=23 total_expected_recv=1131230 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268160 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:03:26] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 637.8 | throughput per GPU (TFLOP/s/GPU): 200.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.119604E+01 | loss scale: 1.0 | grad norm: 396.977 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47646 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47706 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47646 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47706 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=61440 expected_recv=46853 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=61440 expected_recv=46853 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48499 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48499 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48192 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=47476 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49326 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48192 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=47476 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49326 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47713 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47713 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47943 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49341 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47943 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49341 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=61440 expected_recv=46151 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=61440 expected_recv=46151 actual_valid_recv=46720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=46720 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49589 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49589 actual_valid_recv=50688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50688 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=61440 expected_recv=45763 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=61440 expected_recv=45763 actual_valid_recv=46720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46720 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49605 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=45746 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49605 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=45746 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=61440 expected_recv=48075 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=61440 expected_recv=48075 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=61440 expected_recv=47277 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=61440 expected_recv=47277 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=61440 expected_recv=45876 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=61440 expected_recv=45876 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49475 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49475 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50688 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46720 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49546 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=61440 expected_recv=45807 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49546 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=61440 expected_recv=45807 actual_valid_recv=46720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46720 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=46081 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=46081 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47690 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47690 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47662 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47662 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47967 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47387 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47967 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47387 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=61440 expected_recv=46414 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=61440 expected_recv=46414 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=47123 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=47123 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48233 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48233 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=61440 expected_recv=46176 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49641 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=61440 expected_recv=46176 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49641 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=23 total_expected_recv=1133970 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264192 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=23 total_expected_recv=1133392 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265856 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=23 total_expected_recv=1135772 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264576 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=23 total_expected_recv=1124312 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274432 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=23 total_expected_recv=1116463 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=281472 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=23 total_expected_recv=1136336 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=262400 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=23 total_expected_recv=1123931 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275072 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=23 total_expected_recv=1139792 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=260352 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:03:26] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 642.1 | throughput per GPU (TFLOP/s/GPU): 198.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.367741E+00 | loss scale: 1.0 | grad norm: 5.090 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48907 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=47434 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48655 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48907 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=47434 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48655 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49621 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=61440 expected_recv=45949 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49621 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49426 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=61440 expected_recv=45949 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49426 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48496 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=46856 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48496 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=46856 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=47654 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=61440 expected_recv=47698 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=47654 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=61440 expected_recv=47698 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48202 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48202 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=61440 expected_recv=47149 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=61440 expected_recv=47149 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48785 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48785 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=46566 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=46566 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=61440 expected_recv=47753 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=61440 expected_recv=47753 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=61440 expected_recv=47601 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=61440 expected_recv=47601 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47491 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47863 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47491 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47863 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=61440 expected_recv=45963 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48902 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=61440 expected_recv=45963 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48902 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=46448 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=46448 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=46856 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=46856 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48496 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48496 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=61440 expected_recv=46357 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=46410 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=46410 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=61440 expected_recv=46357 actual_valid_recv=46976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46976 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=61440 expected_recv=48994 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=61440 expected_recv=48994 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48687 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=61440 expected_recv=46664 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48687 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=61440 expected_recv=46664 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48371 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48371 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=61440 expected_recv=46977 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=61440 expected_recv=46977 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=45706 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48866 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48866 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=61440 expected_recv=46484 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=61440 expected_recv=46484 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=48375 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=46980 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=48375 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=46980 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=23 total_expected_recv=1128842 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=23 total_expected_recv=1133099 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267776 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=23 total_expected_recv=1136167 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264192 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=23 total_expected_recv=1127840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271744 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=23 total_expected_recv=1125408 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274816 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=23 total_expected_recv=1136554 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=262016 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=23 total_expected_recv=1120414 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=277888 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=23 total_expected_recv=1135644 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263424 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:03:27] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 641.4 | throughput per GPU (TFLOP/s/GPU): 198.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.973585E+00 | loss scale: 1.0 | grad norm: 3.690 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=47787 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48198 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=47787 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48198 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=47354 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=47354 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=48741 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=48741 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48887 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=61440 expected_recv=47341 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48887 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=61440 expected_recv=47341 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48768 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48768 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47922 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47922 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47430 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47430 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=61440 expected_recv=45710 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=61440 expected_recv=45710 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=47567 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=47567 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48973 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48456 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48973 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48456 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48276 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48276 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=61440 expected_recv=47079 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=61440 expected_recv=47079 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49641 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49641 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49647 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49647 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=45711 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=45711 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48281 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=47071 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48281 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=47071 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48155 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48155 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=47199 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=47199 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=61440 expected_recv=45710 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=61440 expected_recv=45710 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48158 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=61440 expected_recv=47154 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48158 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=61440 expected_recv=47154 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48199 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=61440 expected_recv=47194 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48199 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=61440 expected_recv=47194 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49053 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=46298 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49588 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=46298 actual_valid_recv=46976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49588 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46976 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=45762 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=45762 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48698 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48698 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=61440 expected_recv=47223 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49392 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=61440 expected_recv=47223 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49392 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=46908 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=46908 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48442 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48442 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=23 total_expected_recv=1125840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275328 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=23 total_expected_recv=1130625 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269952 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=23 total_expected_recv=1135536 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264832 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=23 total_expected_recv=1123706 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=277120 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=23 total_expected_recv=1133176 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267008 peak_demand=49647 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=23 total_expected_recv=1133689 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268416 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=23 total_expected_recv=1134084 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265088 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=23 total_expected_recv=1127312 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273408 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:03:28] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 639.3 | throughput per GPU (TFLOP/s/GPU): 199.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.787342E+00 | loss scale: 1.0 | grad norm: 5.890 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48474 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48143 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48379 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48474 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48143 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48379 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48741 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48741 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=61440 expected_recv=46610 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=61440 expected_recv=46610 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48310 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48310 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49209 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49209 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=61440 expected_recv=47479 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=61440 expected_recv=47479 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48430 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48073 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48430 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48073 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48493 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48493 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=45861 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=45861 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=47106 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48246 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=47106 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48246 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=61440 expected_recv=48353 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=61440 expected_recv=48353 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=61440 expected_recv=46998 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=61440 expected_recv=46998 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=45711 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=45711 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47508 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=46818 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47508 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=46818 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=61440 expected_recv=48535 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47844 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=61440 expected_recv=48535 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47844 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=23 total_expected_recv=1128939 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=23 total_expected_recv=1134425 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268416 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=23 total_expected_recv=1128332 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271872 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=23 total_expected_recv=1134077 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267008 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=23 total_expected_recv=1125663 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275584 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=23 total_expected_recv=1132966 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=266240 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=23 total_expected_recv=1121690 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=278784 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=23 total_expected_recv=1137876 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263296 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 15:03:28] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 645.8 | throughput per GPU (TFLOP/s/GPU): 197.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.612353E+00 | loss scale: 1.0 | grad norm: 3.038 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48705 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=61440 expected_recv=47832 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48459 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48705 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=61440 expected_recv=47832 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48459 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48620 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=61440 expected_recv=47652 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48620 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=61440 expected_recv=47652 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48724 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48724 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48324 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48324 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=61440 expected_recv=47554 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=61440 expected_recv=47554 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49121 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48460 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48460 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48467 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48069 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48467 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48069 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=61440 expected_recv=46417 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=61440 expected_recv=46417 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=61440 expected_recv=47776 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=61440 expected_recv=47776 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=48131 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49089 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=48131 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=61440 expected_recv=46977 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=61440 expected_recv=46977 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48376 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48376 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=47642 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=47642 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=47710 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=47710 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=61440 expected_recv=47311 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=61440 expected_recv=47311 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48041 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48391 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48041 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48391 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48212 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48394 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48212 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48394 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=46300 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=46300 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=46236 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=46236 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49114 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48510 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49114 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48510 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=47179 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=47179 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49306 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49306 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=45706 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=61440 expected_recv=46443 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=61440 expected_recv=46443 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48908 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48908 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47415 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47415 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47938 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47938 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=23 total_expected_recv=1129820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=23 total_expected_recv=1131823 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268160 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=23 total_expected_recv=1127932 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=23 total_expected_recv=1122831 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=276608 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=23 total_expected_recv=1135130 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264064 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=23 total_expected_recv=1127684 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272640 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=23 total_expected_recv=1134577 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265088 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=23 total_expected_recv=1134171 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265088 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:03:29] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 640.7 | throughput per GPU (TFLOP/s/GPU): 199.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.076241E+00 | loss scale: 1.0 | grad norm: 29.812 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=47764 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48306 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=47764 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48306 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48666 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48666 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=47614 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=47614 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47698 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48714 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47698 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48714 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47654 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47654 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=61440 expected_recv=47848 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=61440 expected_recv=47848 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49611 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49611 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48461 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48461 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48720 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48720 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48213 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=47632 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48213 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=47632 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48146 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48146 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48715 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48135 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48715 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48135 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=61440 expected_recv=47724 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=61440 expected_recv=47724 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=47628 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=47628 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=61440 expected_recv=47364 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=47988 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=61440 expected_recv=47364 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=47988 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48636 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=46718 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48636 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=46718 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47571 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47571 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47782 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47782 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=45804 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=45804 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=61440 expected_recv=46774 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=61440 expected_recv=46774 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49547 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49547 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49130 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49130 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=46221 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=46221 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47700 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47700 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47651 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=61440 expected_recv=47442 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47651 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=61440 expected_recv=47910 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=61440 expected_recv=47442 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=61440 expected_recv=47910 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=61440 expected_recv=47553 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=61440 expected_recv=47553 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=47796 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=47796 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=23 total_expected_recv=1128868 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=23 total_expected_recv=1131367 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268160 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=23 total_expected_recv=1136935 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263168 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=23 total_expected_recv=1127801 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=23 total_expected_recv=1130291 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=23 total_expected_recv=1129916 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269056 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=23 total_expected_recv=1128650 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271104 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=23 total_expected_recv=1130140 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:03:29] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 644.8 | throughput per GPU (TFLOP/s/GPU): 197.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.639501E+00 | loss scale: 1.0 | grad norm: 64.594 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48125 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48853 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48695 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48125 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48853 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48695 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49549 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47814 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49549 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47633 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47814 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47633 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48449 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48449 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=46904 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=46904 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=47193 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=47193 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48159 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48159 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48491 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48665 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=61440 expected_recv=47840 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48491 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48665 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=61440 expected_recv=47840 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=61440 expected_recv=46691 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48853 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49452 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=61440 expected_recv=46691 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48853 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49452 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=47665 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=61440 expected_recv=47687 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=47665 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=61440 expected_recv=47687 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47965 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47882 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47965 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47882 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49381 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49381 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49036 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=61440 expected_recv=46579 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=61440 expected_recv=46579 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48043 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=47497 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48043 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=47497 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=61440 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=61440 expected_recv=45706 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48523 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=46829 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48523 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=46829 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=61440 expected_recv=47705 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=61440 expected_recv=47646 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=61440 expected_recv=47705 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=61440 expected_recv=47646 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=61440 expected_recv=46365 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=61440 expected_recv=46365 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=23 total_expected_recv=1132031 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268288 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=23 total_expected_recv=1130941 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268544 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=23 total_expected_recv=1124233 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274304 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=23 total_expected_recv=1134243 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264832 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=23 total_expected_recv=1131026 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269056 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=23 total_expected_recv=1135171 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264064 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=23 total_expected_recv=1127543 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272384 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=23 total_expected_recv=1128780 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270976 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 15:03:30] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 639.8 | throughput per GPU (TFLOP/s/GPU): 199.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.624613E+00 | loss scale: 1.0 | grad norm: 71.371 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +[after training is done] datetime: 2026-08-26 15:03:30 +[rank0]:[W826 15:03:31.853536302 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 15:03:33.391595756 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 15:03:33.393028686 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 15:03:33.394445923 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 15:03:33.396195529 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 15:03:33.399561398 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 15:03:33.405427623 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 15:03:34.068342075 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:03:35.733000 54788 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:03:35.733000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54878 closing signal SIGTERM +W0826 15:03:35.734000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54879 closing signal SIGTERM +W0826 15:03:35.734000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54880 closing signal SIGTERM +W0826 15:03:35.735000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54881 closing signal SIGTERM +W0826 15:03:35.735000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54882 closing signal SIGTERM +W0826 15:03:35.736000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54883 closing signal SIGTERM +W0826 15:03:35.736000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54884 closing signal SIGTERM +W0826 15:03:35.736000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54885 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 54788 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..20f4b0a3d693a2f5a3851606effe2433ad211900 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,152 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 629 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 2393 +1, 2387 +2, 2371 +3, 2385 +4, 2385 +5, 2397 +6, 2393 +7, 2377 +0, 46703 +1, 46137 +2, 46137 +3, 46837 +4, 46837 +5, 46837 +6, 46837 +7, 46837 +0, 47107 +1, 46537 +2, 46537 +3, 47237 +4, 47237 +5, 47237 +6, 47237 +7, 47237 +0, 47497 +1, 46927 +2, 46927 +3, 47627 +4, 47627 +5, 47627 +6, 47627 +7, 47627 +0, 47497 +1, 46927 +2, 46927 +3, 47627 +4, 47627 +5, 47627 +6, 47627 +7, 47627 +0, 47497 +1, 46927 +2, 46927 +3, 47627 +4, 47627 +5, 47627 +6, 47627 +7, 47627 +0, 47497 +1, 46927 +2, 46927 +3, 47627 +4, 47627 +5, 47627 +6, 47627 +7, 47627 +0, 47497 +1, 46927 +2, 46927 +3, 47627 +4, 47627 +5, 47627 +6, 47627 +7, 47627 +0, 46851 +1, 46345 +2, 46927 +3, 47301 +4, 47045 +5, 47627 +6, 47301 +7, 46981 +0, 2561 +1, 2561 +2, 2561 +3, 2625 +4, 46981 +5, 2884 +6, 2564 +7, 2564 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..6752517ec4555ac82a4ae485c93c65fb9dcba003 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype= +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=512 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ece45ef7ca9a2e8c4e3894574ed214e42055905d --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log @@ -0,0 +1,1884 @@ +[15:04:30] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + QWEN2_57B / ultraep (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 15:04:31.818000 60221 torch/distributed/run.py:766] +W0826 15:04:31.818000 60221 torch/distributed/run.py:766] ***************************************** +W0826 15:04:31.818000 60221 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:04:31.818000 60221 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank [Gloo] Rank [Gloo] Rank 013 is connected to is connected to is connected to 77[Gloo] Rank 7 peer ranks. peer ranks. 2 peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : is connected to Expected number of connected peer ranks is : 7777 + peer ranks. + +Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 75 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 32 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W826 15:04:49.719767567 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:04:49.721747479 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:04:49.721811630 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:04:49.721828016 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:04:49.723746853 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:04:49.726046387 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:04:49.727355538 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 15:04:49.867202970 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.798 seconds +time to initialize megatron (seconds): 7.096 +[after megatron is initialized] datetime: 2026-08-26 15:04:55 +building GPT model ... +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 7262624256 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:04:59 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 15:04:59 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3452.85, 3471.97) + train/valid/test-data-iterators-setup ..........: (70.81, 86.46) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 15:04:59 +WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] +WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] +WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] +WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] +WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] +WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] +WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] +WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] +[rank4]: Traceback (most recent call last): +[rank4]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank4]: pretrain( +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank4]: iteration, num_floating_point_operations_so_far = train( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank4]: ) = train_step( +[rank4]: ^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank4]: losses_reduced = forward_backward_func( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank4]: output_tensor, num_tokens = forward_step( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank4]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank4]: output_tensor = model( +[rank4]: ^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank4]: return self.module(*inputs, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank4]: outputs = self.module(*inputs, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank4]: hidden_states = self.decoder( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank4]: hidden_states, context = layer( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank4]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank4]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank4]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank4]: return self.ultraep_forward(hidden_states) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward +[rank4]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute +[rank4]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute +[rank4]: expanded_probs, expanded_routing_map = self._dense_reroute( +[rank4]: ^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute +[rank4]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank4]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward +[rank4]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +[rank7]: Traceback (most recent call last): +[rank7]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank7]: pretrain( +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank7]: iteration, num_floating_point_operations_so_far = train( +[rank7]: ^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank7]: ) = train_step( +[rank7]: ^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank7]: losses_reduced = forward_backward_func( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank7]: output_tensor, num_tokens = forward_step( +[rank7]: ^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank7]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank7]: output_tensor = model( +[rank7]: ^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank7]: return self.module(*inputs, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank7]: outputs = self.module(*inputs, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank7]: hidden_states = self.decoder( +[rank7]: ^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank7]: hidden_states, context = layer( +[rank7]: ^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank7]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank7]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank7]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank7]: return self.ultraep_forward(hidden_states) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward +[rank7]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute +[rank7]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute +[rank7]: expanded_probs, expanded_routing_map = self._dense_reroute( +[rank7]: ^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute +[rank7]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank7]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward +[rank7]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +[rank1]: Traceback (most recent call last): +[rank1]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank1]: pretrain( +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank1]: ) = train_step( +[rank1]: ^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank1]: output_tensor = model( +[rank1]: ^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank1]: hidden_states = self.decoder( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank1]: hidden_states, context = layer( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank1]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank1]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank1]: return self.ultraep_forward(hidden_states) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward +[rank1]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute +[rank1]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute +[rank1]: expanded_probs, expanded_routing_map = self._dense_reroute( +[rank1]: ^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute +[rank1]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward +[rank1]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +[rank3]: Traceback (most recent call last): +[rank3]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank3]: pretrain( +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank3]: iteration, num_floating_point_operations_so_far = train( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank3]: ) = train_step( +[rank3]: ^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank3]: losses_reduced = forward_backward_func( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank3]: output_tensor, num_tokens = forward_step( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank3]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank3]: output_tensor = model( +[rank3]: ^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank3]: return self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank3]: outputs = self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank3]: hidden_states = self.decoder( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank3]: hidden_states, context = layer( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank3]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank3]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank3]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank3]: return self.ultraep_forward(hidden_states) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward +[rank3]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute +[rank3]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute +[rank3]: expanded_probs, expanded_routing_map = self._dense_reroute( +[rank3]: ^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute +[rank3]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank3]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward +[rank3]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +[rank2]: Traceback (most recent call last): +[rank2]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank2]: pretrain( +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank2]: iteration, num_floating_point_operations_so_far = train( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank2]: ) = train_step( +[rank2]: ^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank2]: losses_reduced = forward_backward_func( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank2]: output_tensor, num_tokens = forward_step( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank2]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank2]: output_tensor = model( +[rank2]: ^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank2]: return self.module(*inputs, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank2]: outputs = self.module(*inputs, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank2]: hidden_states = self.decoder( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank2]: hidden_states, context = layer( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank2]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank2]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank2]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank2]: return self.ultraep_forward(hidden_states) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward +[rank2]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute +[rank2]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute +[rank2]: expanded_probs, expanded_routing_map = self._dense_reroute( +[rank2]: ^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute +[rank2]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank2]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward +[rank2]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +[rank5]: Traceback (most recent call last): +[rank5]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank5]: pretrain( +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank5]: iteration, num_floating_point_operations_so_far = train( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank5]: ) = train_step( +[rank5]: ^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank5]: losses_reduced = forward_backward_func( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank5]: output_tensor, num_tokens = forward_step( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank5]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank5]: output_tensor = model( +[rank5]: ^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank5]: return self.module(*inputs, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank5]: outputs = self.module(*inputs, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank5]: hidden_states = self.decoder( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank5]: hidden_states, context = layer( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank5]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank5]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank5]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank5]: return self.ultraep_forward(hidden_states) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward +[rank5]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute +[rank5]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute +[rank5]: expanded_probs, expanded_routing_map = self._dense_reroute( +[rank5]: ^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute +[rank5]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank5]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward +[rank5]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +[rank6]: Traceback (most recent call last): +[rank6]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank6]: pretrain( +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank6]: iteration, num_floating_point_operations_so_far = train( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank6]: ) = train_step( +[rank6]: ^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank6]: losses_reduced = forward_backward_func( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank6]: output_tensor, num_tokens = forward_step( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank6]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank6]: output_tensor = model( +[rank6]: ^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank6]: return self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank6]: outputs = self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank6]: hidden_states = self.decoder( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank6]: hidden_states, context = layer( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank6]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank6]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank6]: return self.ultraep_forward(hidden_states) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward +[rank6]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute +[rank6]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute +[rank6]: expanded_probs, expanded_routing_map = self._dense_reroute( +[rank6]: ^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute +[rank6]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward +[rank6]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +[rank0]: Traceback (most recent call last): +[rank0]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank0]: pretrain( +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: ^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank0]: ) = train_step( +[rank0]: ^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank0]: output_tensor = model( +[rank0]: ^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank0]: hidden_states = self.decoder( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank0]: hidden_states, context = layer( +[rank0]: ^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank0]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank0]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank0]: return self.ultraep_forward(hidden_states) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward +[rank0]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute +[rank0]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute +[rank0]: expanded_probs, expanded_routing_map = self._dense_reroute( +[rank0]: ^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute +[rank0]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward +[rank0]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 +[rank0]:[W826 15:05:13.484711782 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:05:14.558000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60311 closing signal SIGTERM +W0826 15:05:14.559000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60313 closing signal SIGTERM +W0826 15:05:14.559000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60314 closing signal SIGTERM +W0826 15:05:14.559000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60315 closing signal SIGTERM +W0826 15:05:14.559000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60316 closing signal SIGTERM +W0826 15:05:14.560000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60317 closing signal SIGTERM +W0826 15:05:14.560000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60318 closing signal SIGTERM +E0826 15:05:16.648000 60221 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 1 (pid: 60312) of binary: /usr/bin/python +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-08-26_15:05:14 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 60312) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..0b31b24606450d6588fe2d8b84bf566a01a3444b --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1/mem_samples.csv @@ -0,0 +1,152 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 1001 +1, 995 +2, 983 +3, 1035 +4, 991 +5, 999 +6, 1011 +7, 1005 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 2423 +1, 2425 +2, 2435 +3, 2429 +4, 2421 +5, 2409 +6, 2423 +7, 2451 +0, 43469 +1, 43469 +2, 43469 +3, 43469 +4, 43469 +5, 43469 +6, 43469 +7, 43469 +0, 51937 +1, 51957 +2, 51907 +3, 51955 +4, 51615 +5, 51937 +6, 51905 +7, 51921 +0, 53661 +1, 53661 +2, 53661 +3, 53661 +4, 53373 +5, 53661 +6, 53661 +7, 53661 +0, 54649 +1, 54649 +2, 54649 +3, 54649 +4, 54361 +5, 54649 +6, 54649 +7, 54649 +0, 54649 +1, 54649 +2, 54649 +3, 54649 +4, 54361 +5, 54649 +6, 54649 +7, 54649 +0, 54649 +1, 54649 +2, 54649 +3, 54649 +4, 54361 +5, 54649 +6, 54649 +7, 54649 +0, 54649 +1, 54649 +2, 54649 +3, 54649 +4, 54361 +5, 54649 +6, 54649 +7, 54649 +0, 54649 +1, 54649 +2, 54649 +3, 54649 +4, 54361 +5, 54649 +6, 54649 +7, 54649 +0, 53403 +1, 53403 +2, 53403 +3, 53403 +4, 53115 +5, 53403 +6, 53403 +7, 3844 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..f8ad4a333e8e01042df9618baa074413ddb2633f --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.02 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=512 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d06eafa815de53c36062105512781b85a53a0152 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1/train.log @@ -0,0 +1,2225 @@ +[14:59:28] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[14:59:28] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.02 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 14:59:29.840000 47943 torch/distributed/run.py:766] +W0826 14:59:29.840000 47943 torch/distributed/run.py:766] ***************************************** +W0826 14:59:29.840000 47943 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 14:59:29.840000 47943 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.02 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 576 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 + + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 37 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 14:59:47.960636569 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 14:59:47.967048933 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 14:59:47.967113212 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 14:59:47.967183427 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 14:59:47.970269235 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 14:59:47.970292763 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 14:59:47.970335135 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 14:59:48.121399627 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.753 seconds +time to initialize megatron (seconds): 6.892 +[after megatron is initialized] datetime: 2026-08-26 14:59:53 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8521882624 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 14:59:57 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 14:59:57 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3589.41, 3604.72) + train/valid/test-data-iterators-setup ..........: (66.30, 84.10) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 14:59:57 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.utils:num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +[rank2]: Traceback (most recent call last): +[rank2]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank2]: pretrain( +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank2]: iteration, num_floating_point_operations_so_far = train( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank2]: ) = train_step( +[rank2]: ^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank2]: losses_reduced = forward_backward_func( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank2]: output_tensor, num_tokens = forward_step( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank2]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank2]: output_tensor = model( +[rank2]: ^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank2]: return self.module(*inputs, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank2]: outputs = self.module(*inputs, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank2]: hidden_states = self.decoder( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank2]: hidden_states, context = layer( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank2]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank2]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank2]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank2]: return self.ultraep_forward(hidden_states) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank2]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank2]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank2]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank2]: hidden_states = self._comm_manager.dispatch( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank2]: ) = hybrid_ep_dispatch( +[rank2]: ^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank2]: return HybridEPDispatch.apply( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank2]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank2]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank2]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank7]: Traceback (most recent call last): +[rank7]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank7]: pretrain( +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank7]: iteration, num_floating_point_operations_so_far = train( +[rank7]: ^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank7]: ) = train_step( +[rank7]: ^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank7]: losses_reduced = forward_backward_func( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank7]: output_tensor, num_tokens = forward_step( +[rank7]: ^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank7]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank7]: output_tensor = model( +[rank7]: ^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank7]: return self.module(*inputs, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank7]: outputs = self.module(*inputs, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank7]: hidden_states = self.decoder( +[rank7]: ^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank7]: hidden_states, context = layer( +[rank7]: ^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank7]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank7]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank7]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank7]: return self.ultraep_forward(hidden_states) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank7]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank7]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank7]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank7]: hidden_states = self._comm_manager.dispatch( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank7]: ) = hybrid_ep_dispatch( +[rank7]: ^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank7]: return HybridEPDispatch.apply( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank7]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank7]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank7]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank0]: Traceback (most recent call last): +[rank0]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank0]: pretrain( +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: ^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank0]: ) = train_step( +[rank0]: ^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank0]: output_tensor = model( +[rank0]: ^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank0]: hidden_states = self.decoder( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank0]: hidden_states, context = layer( +[rank0]: ^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank0]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank0]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank0]: return self.ultraep_forward(hidden_states) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank0]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank0]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank0]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank0]: hidden_states = self._comm_manager.dispatch( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank0]: ) = hybrid_ep_dispatch( +[rank0]: ^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank0]: return HybridEPDispatch.apply( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank0]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank0]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank1]: Traceback (most recent call last): +[rank1]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank1]: pretrain( +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank1]: ) = train_step( +[rank1]: ^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank1]: output_tensor = model( +[rank1]: ^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank1]: hidden_states = self.decoder( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank1]: hidden_states, context = layer( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank1]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank1]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank1]: return self.ultraep_forward(hidden_states) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank1]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank1]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank1]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank1]: hidden_states = self._comm_manager.dispatch( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank1]: ) = hybrid_ep_dispatch( +[rank1]: ^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank1]: return HybridEPDispatch.apply( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank1]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank1]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank6]: Traceback (most recent call last): +[rank6]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank6]: pretrain( +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank6]: iteration, num_floating_point_operations_so_far = train( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank6]: ) = train_step( +[rank6]: ^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank6]: losses_reduced = forward_backward_func( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank6]: output_tensor, num_tokens = forward_step( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank6]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank6]: output_tensor = model( +[rank6]: ^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank6]: return self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank6]: outputs = self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank6]: hidden_states = self.decoder( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank6]: hidden_states, context = layer( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank6]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank6]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank6]: return self.ultraep_forward(hidden_states) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank6]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank6]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank6]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank6]: hidden_states = self._comm_manager.dispatch( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank6]: ) = hybrid_ep_dispatch( +[rank6]: ^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank6]: return HybridEPDispatch.apply( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank6]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank6]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank4]: Traceback (most recent call last): +[rank4]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank4]: pretrain( +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank4]: iteration, num_floating_point_operations_so_far = train( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank4]: ) = train_step( +[rank4]: ^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank4]: losses_reduced = forward_backward_func( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank4]: output_tensor, num_tokens = forward_step( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank4]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank4]: output_tensor = model( +[rank4]: ^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank4]: return self.module(*inputs, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank4]: outputs = self.module(*inputs, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank4]: hidden_states = self.decoder( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank4]: hidden_states, context = layer( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank4]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank4]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank4]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank4]: return self.ultraep_forward(hidden_states) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank4]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank4]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank4]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank4]: hidden_states = self._comm_manager.dispatch( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank4]: ) = hybrid_ep_dispatch( +[rank4]: ^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank4]: return HybridEPDispatch.apply( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank4]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank4]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank4]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank5]: Traceback (most recent call last): +[rank5]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank5]: pretrain( +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank5]: iteration, num_floating_point_operations_so_far = train( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank5]: ) = train_step( +[rank5]: ^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank5]: losses_reduced = forward_backward_func( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank5]: output_tensor, num_tokens = forward_step( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank5]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank5]: output_tensor = model( +[rank5]: ^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank5]: return self.module(*inputs, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank5]: outputs = self.module(*inputs, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank5]: hidden_states = self.decoder( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank5]: hidden_states, context = layer( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank5]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank5]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank5]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank5]: return self.ultraep_forward(hidden_states) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank5]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank5]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank5]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank5]: hidden_states = self._comm_manager.dispatch( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank5]: ) = hybrid_ep_dispatch( +[rank5]: ^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank5]: return HybridEPDispatch.apply( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank5]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank5]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank5]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank3]: Traceback (most recent call last): +[rank3]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank3]: pretrain( +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank3]: iteration, num_floating_point_operations_so_far = train( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank3]: ) = train_step( +[rank3]: ^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank3]: losses_reduced = forward_backward_func( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank3]: output_tensor, num_tokens = forward_step( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank3]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank3]: output_tensor = model( +[rank3]: ^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank3]: return self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank3]: outputs = self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank3]: hidden_states = self.decoder( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank3]: hidden_states, context = layer( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank3]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank3]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank3]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank3]: return self.ultraep_forward(hidden_states) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank3]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank3]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank3]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank3]: hidden_states = self._comm_manager.dispatch( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank3]: ) = hybrid_ep_dispatch( +[rank3]: ^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank3]: return HybridEPDispatch.apply( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank3]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank3]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank3]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: AssertionError: num_permuted_tokens (66846) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank0]:[W826 15:00:13.491808138 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:00:14.383000 47943 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 48036 closing signal SIGTERM +W0826 15:00:14.384000 47943 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 48037 closing signal SIGTERM +W0826 15:00:14.385000 47943 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 48038 closing signal SIGTERM +W0826 15:00:14.386000 47943 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 48039 closing signal SIGTERM +W0826 15:00:14.387000 47943 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 48040 closing signal SIGTERM +W0826 15:00:14.387000 47943 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 48041 closing signal SIGTERM +W0826 15:00:14.388000 47943 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 48042 closing signal SIGTERM +E0826 15:00:17.133000 47943 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 7 (pid: 48043) of binary: /usr/bin/python +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-08-26_15:00:14 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 7 (local_rank: 7) + exitcode : 1 (pid: 48043) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..a511e023cab4401e71f166d970318fad75cb2cba --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,240 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 629 +4, 4 +5, 629 +6, 629 +7, 4 +0, 725 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 725 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 821 +1, 819 +2, 819 +3, 821 +4, 819 +5, 821 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2551 +1, 2551 +2, 2551 +3, 2551 +4, 2551 +5, 2551 +6, 2551 +7, 2551 +0, 2959 +1, 2959 +2, 2959 +3, 2959 +4, 2959 +5, 2959 +6, 2959 +7, 2959 +0, 51537 +1, 51537 +2, 51537 +3, 51537 +4, 51249 +5, 51537 +6, 51537 +7, 51537 +0, 52789 +1, 52789 +2, 52789 +3, 52789 +4, 52501 +5, 52789 +6, 52791 +7, 52789 +0, 53651 +1, 53631 +2, 53647 +3, 53643 +4, 53349 +5, 53629 +6, 53633 +7, 53625 +0, 54649 +1, 54649 +2, 54649 +3, 54649 +4, 54361 +5, 54649 +6, 54649 +7, 54649 +0, 55289 +1, 55289 +2, 55289 +3, 55289 +4, 55001 +5, 55289 +6, 55289 +7, 55289 +0, 57201 +1, 57201 +2, 57201 +3, 57201 +4, 56913 +5, 57201 +6, 57201 +7, 57201 +0, 57201 +1, 57201 +2, 57201 +3, 57201 +4, 56913 +5, 57201 +6, 57201 +7, 57201 +0, 90609 +1, 90671 +2, 90681 +3, 90681 +4, 90613 +5, 90667 +6, 90665 +7, 90647 +0, 91083 +1, 91131 +2, 91131 +3, 91131 +4, 91123 +5, 91131 +6, 91131 +7, 91131 +0, 91083 +1, 91131 +2, 91131 +3, 91131 +4, 91125 +5, 91131 +6, 91131 +7, 91131 +0, 91083 +1, 91131 +2, 91131 +3, 91131 +4, 91125 +5, 91131 +6, 91131 +7, 91131 +0, 91375 +1, 91441 +2, 91463 +3, 91479 +4, 91453 +5, 91427 +6, 91417 +7, 91429 +0, 95881 +1, 95959 +2, 95935 +3, 95965 +4, 96031 +5, 95921 +6, 95991 +7, 95941 +0, 122617 +1, 122751 +2, 122751 +3, 122751 +4, 122655 +5, 122751 +6, 122751 +7, 122751 +0, 122617 +1, 122751 +2, 122751 +3, 122751 +4, 122655 +5, 122751 +6, 122751 +7, 122751 +0, 122617 +1, 122751 +2, 122751 +3, 122751 +4, 122655 +5, 122751 +6, 122751 +7, 122751 +0, 5277 +1, 3678 +2, 3678 +3, 3678 +4, 3678 +5, 3678 +6, 3998 +7, 122749 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..39d8f6bf548d96d6494e5b3a5b6a6d9feac05830 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=512 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5d22cd504e81df888c4a8196c2e98835d8153b60 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log @@ -0,0 +1,7317 @@ +[14:58:13] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[14:58:13] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 14:58:14.393000 44602 torch/distributed/run.py:766] +W0826 14:58:14.393000 44602 torch/distributed/run.py:766] ***************************************** +W0826 14:58:14.393000 44602 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 14:58:14.393000 44602 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3[Gloo] Rank is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 77 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W826 14:58:32.833865692 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 14:58:32.833935134 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 14:58:32.833955635 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 14:58:32.834270641 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 14:58:32.834296419 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 14:58:32.838303487 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 14:58:32.838344680 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +g++ -O3 -Wall -shared -std=c++11 -fPIC -fdiagnostics-color -I/usr/include/python3.12 -I/usr/local/lib/python3.12/dist-packages/pybind11/include helpers.cpp -o helpers_cpp.cpython-312-x86_64-linux-gnu.so +In file included from helpers.cpp:12: +In function ‘pybind11::ssize_t pybind11::detail::byte_offset_unsafe(const Strides&, pybind11::ssize_t, Ix ...) [with long int Dim = 0; Strides = std::array; Ix = {}]’, + inlined from ‘const T& pybind11::detail::unchecked_reference::operator()(Ix ...) const [with Ix = {long int}; T = long int; long int Dims = 1]’ at /usr/local/lib/python3.12/dist-packages/pybind11/include/pybind11/numpy.h:545:65, + inlined from ‘const T& pybind11::detail::unchecked_reference::operator[](pybind11::ssize_t) const [with long int D = 1; = void; T = long int; long int Dims = 1]’ at /usr/local/lib/python3.12/dist-packages/pybind11/include/pybind11/numpy.h:553:26, + inlined from ‘void build_exhaustive_blending_indices(pybind11::array_t&, pybind11::array_t&, const pybind11::array_t&, int32_t)’ at helpers.cpp:67:31: +/usr/local/lib/python3.12/dist-packages/pybind11/include/pybind11/numpy.h:494:14: warning: ‘error_argmax’ may be used uninitialized [-Wmaybe-uninitialized] + 494 | return i * strides[Dim] + byte_offset_unsafe(strides, index...); + | ~~^~~~~~~~~~ +helpers.cpp: In function ‘void build_exhaustive_blending_indices(pybind11::array_t&, pybind11::array_t&, const pybind11::array_t&, int32_t)’: +helpers.cpp:49:13: note: ‘error_argmax’ was declared here + 49 | int64_t error_argmax; + | ^~~~~~~~~~~~ +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 5.101 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 14:58:38.068928693 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.714 seconds +time to initialize megatron (seconds): 12.124 +[after megatron is initialized] datetime: 2026-08-26 14:58:43 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8521882624 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.k_layernorm.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 14:58:47 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 14:58:47 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3740.09, 3748.14) + train/valid/test-data-iterators-setup ..........: (69.67, 84.77) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 14:58:47 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63972 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=81920 expected_recv=64773 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=81920 expected_recv=64773 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63972 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=81920 expected_recv=62362 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=81920 expected_recv=62362 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=81920 expected_recv=64152 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=81920 expected_recv=64152 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=81920 expected_recv=62647 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63164 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=81920 expected_recv=62647 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63164 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=81920 expected_recv=64487 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=81920 expected_recv=64487 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=81920 expected_recv=61713 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65584 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=81920 expected_recv=61713 actual_valid_recv=62848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=62848 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65423 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=81920 expected_recv=62983 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65423 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=81920 expected_recv=62983 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66029 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66029 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=81920 expected_recv=62767 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=81920 expected_recv=62767 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=81920 expected_recv=61553 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=81920 expected_recv=61553 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=81920 expected_recv=62939 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=81920 expected_recv=62939 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65000 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=81920 expected_recv=62137 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65000 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=81920 expected_recv=62137 actual_valid_recv=63232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63232 slack_slots=18688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=81920 expected_recv=64368 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=81920 expected_recv=64368 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=81920 expected_recv=64355 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=81920 expected_recv=64355 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=81920 expected_recv=61468 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=81920 expected_recv=61468 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65669 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65669 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=81920 expected_recv=64360 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=81920 expected_recv=64360 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=81920 expected_recv=64291 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=81920 expected_recv=64291 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=81920 expected_recv=62845 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=81920 expected_recv=62845 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=81920 expected_recv=62782 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=81920 expected_recv=62782 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=12 total_expected_recv=792885 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=175360 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=12 total_expected_recv=788470 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=180992 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=12 total_expected_recv=782949 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=12 total_expected_recv=793693 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=174976 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=12 total_expected_recv=785029 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183424 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=12 total_expected_recv=781186 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186752 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=12 total_expected_recv=784940 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184576 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=12 total_expected_recv=782304 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186368 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 14:59:18] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 30537.2 | throughput per GPU (TFLOP/s/GPU): 5.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116001E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67456 | min safe capacity: 1.0293 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 73653.302734375 | max allocated: 80889.87255859375 | reserved: 86236.0 | max reserved: 86236.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63247 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63889 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63247 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63889 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=81920 expected_recv=63648 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=81920 expected_recv=63488 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=81920 expected_recv=63648 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=81920 expected_recv=63488 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67968 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=81920 expected_recv=62987 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=81920 expected_recv=62987 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=81920 expected_recv=64638 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65703 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=81920 expected_recv=64638 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65258 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65258 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=81920 expected_recv=61875 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=81920 expected_recv=61875 actual_valid_recv=63616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65038 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=81920 expected_recv=62098 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65038 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=81920 expected_recv=62098 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63447 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63447 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63690 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63690 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66016 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66016 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=81920 expected_recv=64139 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=81920 expected_recv=64139 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=81920 expected_recv=63173 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=81920 expected_recv=62621 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=81920 expected_recv=63173 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=81920 expected_recv=62621 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=81920 expected_recv=64514 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=81920 expected_recv=64514 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65920 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=81920 expected_recv=61216 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65920 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=81920 expected_recv=61216 actual_valid_recv=62208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=62080 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=62208 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=81920 expected_recv=60943 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=81920 expected_recv=60943 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=12 total_expected_recv=782364 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186880 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=12 total_expected_recv=790871 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=177280 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=12 total_expected_recv=785540 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182912 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=12 total_expected_recv=787276 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182784 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=12 total_expected_recv=792661 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=176896 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=12 total_expected_recv=784796 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183936 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=12 total_expected_recv=784886 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183040 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=12 total_expected_recv=783062 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184064 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 14:59:18] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 496.9 | throughput per GPU (TFLOP/s/GPU): 363.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117607E+01 | loss scale: 1.0 | grad norm: 44.341 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67712 | min safe capacity: 1.0332 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=81920 expected_recv=60945 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=81920 expected_recv=60945 actual_valid_recv=61824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=61824 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=81920 expected_recv=64036 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=81920 expected_recv=64036 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=81920 expected_recv=63099 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=81920 expected_recv=63099 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=81920 expected_recv=62372 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=81920 expected_recv=64764 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=81920 expected_recv=62372 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=81920 expected_recv=64764 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63740 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63740 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63396 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63396 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=81920 expected_recv=62784 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=81920 expected_recv=62784 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=81920 expected_recv=64542 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66002 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=81920 expected_recv=64542 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66002 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=62208 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=81920 expected_recv=60945 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=81920 expected_recv=60945 actual_valid_recv=62208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=62208 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63241 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63895 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63241 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63895 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66057 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66057 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=81920 expected_recv=63362 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=81920 expected_recv=63362 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65746 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=81920 expected_recv=64356 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65746 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=81920 expected_recv=64356 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=81920 expected_recv=63503 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=81920 expected_recv=64776 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=81920 expected_recv=63503 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=81920 expected_recv=64776 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65049 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65049 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=12 total_expected_recv=773216 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193792 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=12 total_expected_recv=790352 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=179200 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=12 total_expected_recv=786334 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=181888 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=12 total_expected_recv=783197 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=12 total_expected_recv=793855 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=173952 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=12 total_expected_recv=790817 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=177408 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=12 total_expected_recv=783618 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184960 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=12 total_expected_recv=790067 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=179072 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 14:59:18] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 341.3 | throughput per GPU (TFLOP/s/GPU): 529.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685495E+00 | loss scale: 1.0 | grad norm: 62.685 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=81920 expected_recv=64727 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=81920 expected_recv=62409 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=81920 expected_recv=64727 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=81920 expected_recv=62409 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=81920 expected_recv=63632 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65361 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=81920 expected_recv=63632 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65286 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65241 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65361 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65286 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65241 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64995 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65480 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66097 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65008 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64995 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66097 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64131 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65480 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65008 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64131 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65939 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=81920 expected_recv=62533 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65939 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=81920 expected_recv=62533 actual_valid_recv=63104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65515 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=63104 slack_slots=18816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64325 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65090 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64325 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65090 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64585 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64585 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65520 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=81920 expected_recv=61600 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=81920 expected_recv=61600 actual_valid_recv=62592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64993 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65836 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64993 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65836 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64343 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64343 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64348 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=81920 expected_recv=64069 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64348 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=81920 expected_recv=64069 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65114 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=81920 expected_recv=64806 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65114 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=81920 expected_recv=64806 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64278 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65573 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65468 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64278 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65573 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65468 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64621 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64621 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65772 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65772 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65385 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=81920 expected_recv=62733 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=81920 expected_recv=62733 actual_valid_recv=63488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65210 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65210 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65360 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65193 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65360 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65193 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=81920 expected_recv=64993 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65235 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=81920 expected_recv=64993 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65235 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65622 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65501 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65101 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66127 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65101 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66127 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=81920 expected_recv=64086 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=81920 expected_recv=64086 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65182 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65216 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65182 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65216 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=12 total_expected_recv=788338 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182272 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=12 total_expected_recv=786775 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183808 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=12 total_expected_recv=786813 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184832 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=12 total_expected_recv=786157 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184960 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=12 total_expected_recv=788487 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183296 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=12 total_expected_recv=782986 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188800 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=12 total_expected_recv=784025 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186624 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=12 total_expected_recv=787875 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=66192 required_capacity=1.010010 + [2026-08-26 14:59:19] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 341.8 | throughput per GPU (TFLOP/s/GPU): 529.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338049E+01 | loss scale: 1.0 | grad norm: 20.321 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67968 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=81920 expected_recv=62587 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=81920 expected_recv=62587 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67968 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65218 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65218 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65574 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65574 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65506 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=81920 expected_recv=64627 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65613 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65239 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65239 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65613 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=81920 expected_recv=64627 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65345 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65345 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=81920 expected_recv=63637 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=81920 expected_recv=64190 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65501 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=81920 expected_recv=63637 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=81920 expected_recv=64190 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=81920 expected_recv=62993 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65460 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=81920 expected_recv=62993 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64539 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64539 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64636 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64811 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64636 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64811 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65566 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=81920 expected_recv=63590 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=81920 expected_recv=63590 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65644 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65386 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65644 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=81920 expected_recv=63844 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65386 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=81920 expected_recv=63844 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65816 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65816 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=81920 expected_recv=64324 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=81920 expected_recv=64324 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65585 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64334 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64334 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65497 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65497 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65481 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64815 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65481 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64815 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65785 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65331 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65785 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65331 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65162 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=81920 expected_recv=63893 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65162 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=81920 expected_recv=63893 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=81920 expected_recv=64318 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65857 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=81920 expected_recv=64318 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65857 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65258 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65513 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65378 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65258 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64545 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65378 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65580 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64545 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64649 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64649 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65677 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66139 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65677 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65360 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66139 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65360 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=12 total_expected_recv=787517 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184832 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=12 total_expected_recv=788126 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183680 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=12 total_expected_recv=789494 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183424 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=12 total_expected_recv=785500 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=12 total_expected_recv=789459 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182912 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=12 total_expected_recv=791724 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=179968 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=12 total_expected_recv=771715 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=201728 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=12 total_expected_recv=787921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=66194 required_capacity=1.010040 + [2026-08-26 14:59:19] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 340.1 | throughput per GPU (TFLOP/s/GPU): 531.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.052464E+01 | loss scale: 1.0 | grad norm: 95.534 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=81920 expected_recv=62846 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=81920 expected_recv=64290 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=81920 expected_recv=62846 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=81920 expected_recv=64290 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=81920 expected_recv=63690 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=81920 expected_recv=64750 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=81920 expected_recv=63690 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=81920 expected_recv=64750 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=81920 expected_recv=63259 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=81920 expected_recv=63259 actual_valid_recv=63616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65595 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65122 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65595 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65122 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64749 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65520 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64749 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64406 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65493 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64406 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64067 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64067 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65490 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64930 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65490 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65696 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64930 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65696 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65985 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64696 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65985 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64696 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64832 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64832 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65351 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65501 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65351 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65512 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65568 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65511 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=81920 expected_recv=63592 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=81920 expected_recv=62307 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=81920 expected_recv=63592 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=81920 expected_recv=62307 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65495 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65495 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65632 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65632 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64862 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64862 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64931 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64931 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65201 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65370 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65201 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65370 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65163 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65511 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65163 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65495 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65598 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65495 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65244 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65244 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=81920 expected_recv=64189 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=81920 expected_recv=64189 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65269 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65261 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65269 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65515 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65261 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=81920 expected_recv=64567 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=81920 expected_recv=64567 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65493 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65629 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65629 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65399 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65399 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=12 total_expected_recv=790933 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184192 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=12 total_expected_recv=782494 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192128 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=12 total_expected_recv=787202 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186112 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=12 total_expected_recv=786923 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187904 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=12 total_expected_recv=782436 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=191872 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=12 total_expected_recv=787835 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185344 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=12 total_expected_recv=788244 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186624 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=12 total_expected_recv=785389 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188800 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 14:59:19] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 340.6 | throughput per GPU (TFLOP/s/GPU): 530.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.028087E+01 | loss scale: 1.0 | grad norm: 41.959 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65100 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=81920 expected_recv=62777 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65100 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65451 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=81920 expected_recv=62777 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67968 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65451 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=81920 expected_recv=62963 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65435 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=81920 expected_recv=62963 actual_valid_recv=63488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65579 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65435 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65579 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66114 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64425 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64878 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66114 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64870 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64425 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64878 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64870 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65425 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65425 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=81920 expected_recv=64208 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65551 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=81920 expected_recv=64208 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64578 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64955 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64578 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64491 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64955 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64491 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65496 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65590 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65590 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65551 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=81920 expected_recv=64077 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65599 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=81920 expected_recv=64077 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65599 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65549 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=81920 expected_recv=63956 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=81920 expected_recv=63956 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65210 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=81920 expected_recv=64818 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65210 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=81920 expected_recv=64818 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=81920 expected_recv=62426 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65391 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=81920 expected_recv=62426 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65391 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65513 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=81920 expected_recv=64916 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65515 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=81920 expected_recv=64916 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66128 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65598 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66128 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=81920 expected_recv=63555 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=81920 expected_recv=63555 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=81920 expected_recv=64959 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65363 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=81920 expected_recv=64959 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65363 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65610 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65610 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65521 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65559 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=81920 expected_recv=64882 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65123 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66054 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=81920 expected_recv=64882 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65123 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66054 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65572 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=81920 expected_recv=64081 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=81920 expected_recv=64081 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65570 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65545 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64761 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65512 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64761 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65550 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64966 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64966 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=12 total_expected_recv=790421 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=12 total_expected_recv=790116 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=12 total_expected_recv=788361 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187520 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=12 total_expected_recv=782428 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194304 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=12 total_expected_recv=788097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187392 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=12 total_expected_recv=783696 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192512 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=12 total_expected_recv=780145 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=196096 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=12 total_expected_recv=788192 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186624 peak_demand=66192 required_capacity=1.010010 + [2026-08-26 14:59:20] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 341.3 | throughput per GPU (TFLOP/s/GPU): 529.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.026134E+01 | loss scale: 1.0 | grad norm: 6.258 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64886 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64974 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64417 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64886 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64974 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65905 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64417 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65905 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65578 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65636 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=81920 expected_recv=64873 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65160 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65578 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65636 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=81920 expected_recv=64873 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65160 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65116 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65116 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64861 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64861 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64886 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64886 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65575 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65575 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=81920 expected_recv=62887 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=81920 expected_recv=62887 actual_valid_recv=63616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=81920 expected_recv=64333 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65748 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=81920 expected_recv=64333 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65748 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65199 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65551 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65267 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65199 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65267 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65547 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65498 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65540 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65528 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65235 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65481 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65235 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65955 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65481 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=81920 expected_recv=63714 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65955 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=81920 expected_recv=63714 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65546 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65620 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65468 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65620 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65468 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=81920 expected_recv=64566 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65264 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=81920 expected_recv=64566 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65264 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65552 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65180 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=81920 expected_recv=64957 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65180 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=81920 expected_recv=64957 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=81920 expected_recv=64506 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=81920 expected_recv=64506 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64862 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64916 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64862 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64916 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64870 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65358 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65682 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64870 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65358 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64995 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65682 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64995 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64807 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64807 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=12 total_expected_recv=784399 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192640 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=12 total_expected_recv=788741 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187136 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=12 total_expected_recv=789933 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187136 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=12 total_expected_recv=785919 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192384 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=12 total_expected_recv=789714 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187264 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=12 total_expected_recv=785165 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192000 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=12 total_expected_recv=783768 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192896 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=12 total_expected_recv=783817 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194304 peak_demand=66192 required_capacity=1.010010 + [2026-08-26 14:59:20] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 339.6 | throughput per GPU (TFLOP/s/GPU): 532.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.796568E+00 | loss scale: 1.0 | grad norm: 5.584 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=81920 expected_recv=60945 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=81920 expected_recv=60945 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=81920 expected_recv=62255 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=81920 expected_recv=62255 actual_valid_recv=62592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65667 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65667 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65131 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65117 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65131 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65117 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=81920 expected_recv=64890 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65550 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=81920 expected_recv=64890 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65309 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65309 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=81920 expected_recv=63782 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=81920 expected_recv=63782 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65482 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65482 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=81920 expected_recv=64118 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65020 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=81920 expected_recv=64118 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65020 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65003 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=81920 expected_recv=64113 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65003 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=81920 expected_recv=64113 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65348 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=81920 expected_recv=64958 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65348 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65065 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=81920 expected_recv=64958 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65065 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65527 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65435 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65435 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64906 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65571 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64906 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65571 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64862 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64862 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=81920 expected_recv=62917 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65553 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=81920 expected_recv=62917 actual_valid_recv=63360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=63360 slack_slots=18560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65934 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=81920 expected_recv=63181 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65934 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=81920 expected_recv=63181 actual_valid_recv=63488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65548 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=81920 expected_recv=64254 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=81920 expected_recv=64254 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=81920 expected_recv=64866 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64519 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=81920 expected_recv=64866 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64519 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64652 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64811 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64652 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64811 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=12 total_expected_recv=789799 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186496 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=12 total_expected_recv=787732 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189440 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=12 total_expected_recv=784868 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192640 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=12 total_expected_recv=783255 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194048 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=12 total_expected_recv=788795 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188800 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=12 total_expected_recv=787830 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189952 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=12 total_expected_recv=785975 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=191616 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=12 total_expected_recv=783202 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193920 peak_demand=66192 required_capacity=1.010010 + [2026-08-26 14:59:21] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 338.8 | throughput per GPU (TFLOP/s/GPU): 533.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.528715E+00 | loss scale: 1.0 | grad norm: 5.450 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=81920 expected_recv=60943 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=81920 expected_recv=60943 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65986 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=81920 expected_recv=63349 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65115 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65737 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65986 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=81920 expected_recv=63349 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65525 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65115 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65737 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64968 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65041 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64540 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65622 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64968 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65041 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64540 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65540 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65575 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65574 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=81920 expected_recv=64006 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65025 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65575 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65574 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=81920 expected_recv=64006 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65025 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65473 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65620 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66156 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65473 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65620 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66156 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=81920 expected_recv=63396 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65726 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=81920 expected_recv=63396 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65726 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65087 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=81920 expected_recv=63982 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65087 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=81920 expected_recv=63982 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65580 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65218 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65218 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66078 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=81920 expected_recv=63806 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65521 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66078 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65743 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=81920 expected_recv=63806 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65743 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65037 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65037 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65511 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65694 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=81920 expected_recv=64592 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65694 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=81920 expected_recv=64592 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=81920 expected_recv=63718 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=81920 expected_recv=64707 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=81920 expected_recv=63718 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=81920 expected_recv=64707 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65553 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=81920 expected_recv=62906 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=81920 expected_recv=62906 actual_valid_recv=63616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65516 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=81920 expected_recv=63618 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=81920 expected_recv=63618 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=81920 expected_recv=64844 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=81920 expected_recv=64844 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=81920 expected_recv=64626 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=81920 expected_recv=64626 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65647 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=81920 expected_recv=64079 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65647 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=81920 expected_recv=64079 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65168 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65168 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=12 total_expected_recv=775742 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=198912 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=12 total_expected_recv=787458 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186880 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=12 total_expected_recv=789723 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=12 total_expected_recv=786339 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188928 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=12 total_expected_recv=789474 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185088 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=12 total_expected_recv=785205 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189568 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=12 total_expected_recv=788230 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188288 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=12 total_expected_recv=789285 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185344 peak_demand=66192 required_capacity=1.010010 + [2026-08-26 14:59:21] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 340.3 | throughput per GPU (TFLOP/s/GPU): 531.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.429726E+00 | loss scale: 1.0 | grad norm: 5.282 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=81920 expected_recv=63722 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=81920 expected_recv=63722 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=81920 expected_recv=63413 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=81920 expected_recv=63413 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65208 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65553 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=81920 expected_recv=64834 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=81920 expected_recv=63923 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65208 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=81920 expected_recv=64834 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=81920 expected_recv=63923 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=81920 expected_recv=64928 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65625 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65021 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=81920 expected_recv=64928 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=81920 expected_recv=64589 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65021 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65625 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=81920 expected_recv=64589 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65549 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=81920 expected_recv=64989 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65676 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=81920 expected_recv=64989 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=81920 expected_recv=63835 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65676 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=81920 expected_recv=63835 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65572 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65640 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64982 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64205 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65260 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64982 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64205 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65724 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65260 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65724 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64204 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64204 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64261 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64261 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=81920 expected_recv=62120 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=81920 expected_recv=62120 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65630 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65577 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65630 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=81920 expected_recv=62911 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=81920 expected_recv=62911 actual_valid_recv=63488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65559 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=81920 expected_recv=64650 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=81920 expected_recv=63774 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=81920 expected_recv=64650 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=81920 expected_recv=63774 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=81920 expected_recv=63047 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=81920 expected_recv=63047 actual_valid_recv=63488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65528 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65389 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65389 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65556 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=81920 expected_recv=64371 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=81920 expected_recv=64371 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65515 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65419 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65419 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=81920 expected_recv=64872 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=81920 expected_recv=64872 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65134 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65548 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65134 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=81920 expected_recv=63875 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=81920 expected_recv=63875 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65621 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=12 total_expected_recv=779399 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=197248 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=12 total_expected_recv=788941 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187904 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=12 total_expected_recv=787163 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190336 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=12 total_expected_recv=785219 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190720 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=12 total_expected_recv=786348 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=191232 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=12 total_expected_recv=790643 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184960 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=12 total_expected_recv=786681 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189952 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=12 total_expected_recv=787062 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190464 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 14:59:21] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 338.2 | throughput per GPU (TFLOP/s/GPU): 534.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.299762E+00 | loss scale: 1.0 | grad norm: 5.041 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=81920 expected_recv=64137 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=81920 expected_recv=62998 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=81920 expected_recv=64137 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=81920 expected_recv=62998 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65329 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66002 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65329 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66002 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=81920 expected_recv=61997 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=81920 expected_recv=61997 actual_valid_recv=62592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65231 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65231 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65715 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=81920 expected_recv=62985 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65715 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=81920 expected_recv=62985 actual_valid_recv=63360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65589 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=63360 slack_slots=18560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=81920 expected_recv=61725 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65411 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=81920 expected_recv=61725 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65411 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=81920 expected_recv=61455 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=81920 expected_recv=61455 actual_valid_recv=62080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65681 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=62080 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65681 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65499 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65499 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=81920 expected_recv=62607 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65222 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=81920 expected_recv=62607 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65222 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=81920 expected_recv=63882 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65613 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=81920 expected_recv=63882 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65613 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=81920 expected_recv=63837 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=81920 expected_recv=63837 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65513 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=81920 expected_recv=63479 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=81920 expected_recv=63479 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=81920 expected_recv=64336 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=81920 expected_recv=61840 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=81920 expected_recv=64336 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=81920 expected_recv=61840 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65933 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65557 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65933 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65094 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65557 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65688 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65094 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65450 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65688 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65450 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65658 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=81920 expected_recv=63822 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=81920 expected_recv=64183 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65658 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65032 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=81920 expected_recv=63822 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=81920 expected_recv=64183 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=81920 expected_recv=64772 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65032 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=81920 expected_recv=64772 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65288 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=81920 expected_recv=64914 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65572 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65288 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=81920 expected_recv=64914 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65075 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65075 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=12 total_expected_recv=784402 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192640 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=12 total_expected_recv=781567 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194304 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=12 total_expected_recv=790921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184320 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=12 total_expected_recv=781388 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=195456 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=12 total_expected_recv=789720 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187008 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=12 total_expected_recv=789980 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187264 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=12 total_expected_recv=786929 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190080 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=12 total_expected_recv=786549 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190080 peak_demand=66192 required_capacity=1.010010 + [2026-08-26 14:59:22] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 338.1 | throughput per GPU (TFLOP/s/GPU): 534.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.177852E+00 | loss scale: 1.0 | grad norm: 4.691 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +[after training is done] datetime: 2026-08-26 14:59:22 +W0826 14:59:23.077000 44602 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 14:59:23.078000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44695 closing signal SIGTERM +W0826 14:59:23.078000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44696 closing signal SIGTERM +W0826 14:59:23.079000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44697 closing signal SIGTERM +W0826 14:59:23.080000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44698 closing signal SIGTERM +W0826 14:59:23.081000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44699 closing signal SIGTERM +W0826 14:59:23.082000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44700 closing signal SIGTERM +W0826 14:59:23.082000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44701 closing signal SIGTERM +W0826 14:59:23.083000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44702 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 44602 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..c9be249873a8791f9f2fcad91381b0cc83b18735 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1/mem_samples.csv @@ -0,0 +1,152 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 26 +2, 629 +3, 629 +4, 27 +5, 27 +6, 629 +7, 27 +0, 925 +1, 923 +2, 933 +3, 947 +4, 935 +5, 927 +6, 915 +7, 915 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 2169 +1, 2181 +2, 2157 +3, 2193 +4, 2169 +5, 2169 +6, 2157 +7, 2171 +0, 11831 +1, 11811 +2, 11711 +3, 11851 +4, 11711 +5, 11511 +6, 12011 +7, 11611 +0, 47005 +1, 47007 +2, 47063 +3, 46997 +4, 47027 +5, 47009 +6, 46995 +7, 47015 +0, 48439 +1, 48445 +2, 48451 +3, 48407 +4, 48441 +5, 48451 +6, 48443 +7, 48469 +0, 48645 +1, 48647 +2, 48657 +3, 48619 +4, 48643 +5, 48657 +6, 48647 +7, 48657 +0, 48645 +1, 48647 +2, 48657 +3, 48619 +4, 48643 +5, 48657 +6, 48647 +7, 48657 +0, 48645 +1, 48647 +2, 48657 +3, 48619 +4, 48643 +5, 48657 +6, 48647 +7, 48657 +0, 48645 +1, 48647 +2, 48657 +3, 48619 +4, 48643 +5, 48657 +6, 48647 +7, 48657 +0, 48645 +1, 48647 +2, 48657 +3, 48619 +4, 48643 +5, 48657 +6, 48647 +7, 48657 +0, 47829 +1, 47831 +2, 47841 +3, 47803 +4, 47827 +5, 47841 +6, 47831 +7, 47841 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..eb64e27724982a9775cdf2a4230a0ff8ef7f5939 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.02 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=256 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8dd62287c45f46e84bc9632f8680414ea971d6e0 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1/train.log @@ -0,0 +1,3649 @@ +[15:01:35] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:01:35] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.02 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 15:01:36.854000 53051 torch/distributed/run.py:766] +W0826 15:01:36.854000 53051 torch/distributed/run.py:766] ***************************************** +W0826 15:01:36.854000 53051 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:01:36.854000 53051 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.02 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 146 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 + + +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 74 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W826 15:01:55.080159103 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:01:55.080580755 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:01:55.080599995 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:01:55.080646974 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:01:55.080646899 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:01:55.080749381 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:01:55.082472437 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 15:01:55.251607361 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.657 seconds +time to initialize megatron (seconds): 6.944 +[after megatron is initialized] datetime: 2026-08-26 15:02:00 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8372566016 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.pre_mlp_layernorm.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.linear_qkv.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.pre_mlp_layernorm.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_qkv.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.router.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:02:05 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 15:02:05 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (4577.16, 4595.00) + train/valid/test-data-iterators-setup ..........: (69.97, 88.21) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 15:02:05 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.utils:num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +WARNING:megatron.core.utils:num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute\n dispatched_input, probs = self.dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch\n return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch\n hidden_states = self._comm_manager.dispatch(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch\n ) = hybrid_ep_dispatch(\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch\n return HybridEPDispatch.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward\n ) = _hybrid_ep_buffer.dispatch_with_permute(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute\n assert num_permuted_tokens % pad_multiple == 0, \\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode.\n'] +[rank7]: Traceback (most recent call last): +[rank7]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank7]: pretrain( +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank7]: iteration, num_floating_point_operations_so_far = train( +[rank7]: ^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank7]: ) = train_step( +[rank7]: ^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank7]: losses_reduced = forward_backward_func( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank7]: output_tensor, num_tokens = forward_step( +[rank7]: ^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank7]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank7]: output_tensor = model( +[rank7]: ^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank7]: return self.module(*inputs, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank7]: outputs = self.module(*inputs, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank7]: hidden_states = self.decoder( +[rank7]: ^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank7]: hidden_states, context = layer( +[rank7]: ^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank7]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank7]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank7]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank7]: return self._call_impl(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank7]: return forward_call(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank7]: return self.ultraep_forward(hidden_states) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank7]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank7]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank7]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank7]: hidden_states = self._comm_manager.dispatch( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank7]: ) = hybrid_ep_dispatch( +[rank7]: ^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank7]: return HybridEPDispatch.apply( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank7]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank7]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank7]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank0]: Traceback (most recent call last): +[rank0]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank0]: pretrain( +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: ^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank0]: ) = train_step( +[rank0]: ^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank0]: output_tensor = model( +[rank0]: ^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank0]: hidden_states = self.decoder( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank0]: hidden_states, context = layer( +[rank0]: ^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank0]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank0]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank0]: return self.ultraep_forward(hidden_states) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank0]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank0]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank0]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank0]: hidden_states = self._comm_manager.dispatch( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank0]: ) = hybrid_ep_dispatch( +[rank0]: ^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank0]: return HybridEPDispatch.apply( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank0]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank0]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank1]: Traceback (most recent call last): +[rank1]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank1]: pretrain( +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank1]: ) = train_step( +[rank1]: ^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank1]: output_tensor = model( +[rank1]: ^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank1]: hidden_states = self.decoder( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank1]: hidden_states, context = layer( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank1]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank1]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank1]: return self.ultraep_forward(hidden_states) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank1]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank1]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank1]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank1]: hidden_states = self._comm_manager.dispatch( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank1]: ) = hybrid_ep_dispatch( +[rank1]: ^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank1]: return HybridEPDispatch.apply( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank1]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank1]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank6]: Traceback (most recent call last): +[rank6]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank6]: pretrain( +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank6]: iteration, num_floating_point_operations_so_far = train( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank6]: ) = train_step( +[rank6]: ^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank6]: losses_reduced = forward_backward_func( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank6]: output_tensor, num_tokens = forward_step( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank6]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank6]: output_tensor = model( +[rank6]: ^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank6]: return self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank6]: outputs = self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank6]: hidden_states = self.decoder( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank6]: hidden_states, context = layer( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank6]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank6]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank6]: return self.ultraep_forward(hidden_states) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank6]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank6]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank6]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank6]: hidden_states = self._comm_manager.dispatch( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank6]: ) = hybrid_ep_dispatch( +[rank6]: ^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank6]: return HybridEPDispatch.apply( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank6]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank6]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank2]: Traceback (most recent call last): +[rank2]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank2]: pretrain( +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank2]: iteration, num_floating_point_operations_so_far = train( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank2]: ) = train_step( +[rank2]: ^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank2]: losses_reduced = forward_backward_func( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank2]: output_tensor, num_tokens = forward_step( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank2]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank2]: output_tensor = model( +[rank2]: ^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank2]: return self.module(*inputs, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank2]: outputs = self.module(*inputs, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank2]: hidden_states = self.decoder( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank2]: hidden_states, context = layer( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank2]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank2]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank2]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank2]: return self._call_impl(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank2]: return forward_call(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank2]: return self.ultraep_forward(hidden_states) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank2]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank2]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank2]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank2]: hidden_states = self._comm_manager.dispatch( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank2]: ) = hybrid_ep_dispatch( +[rank2]: ^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank2]: return HybridEPDispatch.apply( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank2]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank2]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank2]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank4]: Traceback (most recent call last): +[rank4]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank4]: pretrain( +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank4]: iteration, num_floating_point_operations_so_far = train( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank4]: ) = train_step( +[rank4]: ^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank4]: losses_reduced = forward_backward_func( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank4]: output_tensor, num_tokens = forward_step( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank4]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank4]: output_tensor = model( +[rank4]: ^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank4]: return self.module(*inputs, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank4]: outputs = self.module(*inputs, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank4]: hidden_states = self.decoder( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank4]: hidden_states, context = layer( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank4]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank4]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank4]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank4]: return self._call_impl(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank4]: return forward_call(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank4]: return self.ultraep_forward(hidden_states) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank4]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank4]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank4]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank4]: hidden_states = self._comm_manager.dispatch( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank4]: ) = hybrid_ep_dispatch( +[rank4]: ^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank4]: return HybridEPDispatch.apply( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank4]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank4]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank4]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank3]: Traceback (most recent call last): +[rank3]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank3]: pretrain( +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank3]: iteration, num_floating_point_operations_so_far = train( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank3]: ) = train_step( +[rank3]: ^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank3]: losses_reduced = forward_backward_func( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank3]: output_tensor, num_tokens = forward_step( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank3]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank3]: output_tensor = model( +[rank3]: ^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank3]: return self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank3]: outputs = self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank3]: hidden_states = self.decoder( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank3]: hidden_states, context = layer( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank3]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank3]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank3]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank3]: return self.ultraep_forward(hidden_states) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank3]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank3]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank3]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank3]: hidden_states = self._comm_manager.dispatch( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank3]: ) = hybrid_ep_dispatch( +[rank3]: ^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank3]: return HybridEPDispatch.apply( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank3]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank3]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank3]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank5]: Traceback (most recent call last): +[rank5]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in +[rank5]: pretrain( +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain +[rank5]: iteration, num_floating_point_operations_so_far = train( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train +[rank5]: ) = train_step( +[rank5]: ^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step +[rank5]: losses_reduced = forward_backward_func( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank5]: output_tensor, num_tokens = forward_step( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank5]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step +[rank5]: output_tensor = model( +[rank5]: ^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank5]: return self.module(*inputs, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward +[rank5]: outputs = self.module(*inputs, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank5]: hidden_states = self.decoder( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank5]: hidden_states, context = layer( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ +[rank5]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank5]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp +[rank5]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward +[rank5]: return self.ultraep_forward(hidden_states) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 608, in ultraep_forward +[rank5]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 588, in dispatch_and_compute +[rank5]: dispatched_input, probs = self.dispatch(hidden_states, probs, metadata) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 638, in dispatch +[rank5]: return self.token_dispatcher.token_dispatch(hidden_states, probs, metadata) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1672, in token_dispatch +[rank5]: hidden_states = self._comm_manager.dispatch( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/token_dispatcher.py", line 1216, in dispatch +[rank5]: ) = hybrid_ep_dispatch( +[rank5]: ^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 685, in hybrid_ep_dispatch +[rank5]: return HybridEPDispatch.apply( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank5]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/fused_a2a.py", line 370, in forward +[rank5]: ) = _hybrid_ep_buffer.dispatch_with_permute( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py", line 357, in dispatch_with_permute +[rank5]: assert num_permuted_tokens % pad_multiple == 0, \ +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: AssertionError: num_permuted_tokens (33423) must be a multiple of pad_multiple (128) in non-blocking mode. +[rank0]:[W826 15:02:20.468180963 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:02:22.097000 53051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 53144 closing signal SIGTERM +W0826 15:02:22.097000 53051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 53145 closing signal SIGTERM +W0826 15:02:22.098000 53051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 53146 closing signal SIGTERM +W0826 15:02:22.098000 53051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 53147 closing signal SIGTERM +W0826 15:02:22.098000 53051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 53148 closing signal SIGTERM +W0826 15:02:22.099000 53051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 53149 closing signal SIGTERM +W0826 15:02:22.099000 53051 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 53151 closing signal SIGTERM +E0826 15:02:24.430000 53051 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 6 (pid: 53150) of binary: /usr/bin/python +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-08-26_15:02:22 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 6 (local_rank: 6) + exitcode : 1 (pid: 53150) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..6178ae913713b67e8a9f0249dd993445927004ed --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 921 +1, 913 +2, 919 +3, 923 +4, 909 +5, 913 +6, 909 +7, 915 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 2217 +1, 2221 +2, 2219 +3, 2221 +4, 2197 +5, 2215 +6, 2209 +7, 2219 +0, 11431 +1, 11511 +2, 11311 +3, 11811 +4, 11711 +5, 11711 +6, 11511 +7, 11511 +0, 47039 +1, 47017 +2, 47057 +3, 47021 +4, 47047 +5, 47059 +6, 47037 +7, 47051 +0, 47829 +1, 47831 +2, 47841 +3, 47803 +4, 47827 +5, 47841 +6, 47831 +7, 47841 +0, 48645 +1, 48647 +2, 48657 +3, 48619 +4, 48643 +5, 48657 +6, 48647 +7, 48657 +0, 48805 +1, 48807 +2, 48817 +3, 48779 +4, 48803 +5, 48817 +6, 48807 +7, 48817 +0, 49285 +1, 49287 +2, 49293 +3, 49259 +4, 49259 +5, 49293 +6, 49243 +7, 49293 +0, 49577 +1, 49579 +2, 49581 +3, 49551 +4, 49551 +5, 49581 +6, 49535 +7, 49581 +0, 85197 +1, 85985 +2, 85195 +3, 85211 +4, 85211 +5, 85195 +6, 85195 +7, 85195 +0, 86697 +1, 86699 +2, 86695 +3, 86711 +4, 86711 +5, 86695 +6, 86695 +7, 86695 +0, 86697 +1, 86699 +2, 86695 +3, 86711 +4, 86711 +5, 86695 +6, 86695 +7, 86695 +0, 86697 +1, 86699 +2, 86695 +3, 86711 +4, 86711 +5, 86695 +6, 86695 +7, 86695 +0, 86709 +1, 86711 +2, 86707 +3, 86723 +4, 86723 +5, 86707 +6, 86707 +7, 86707 +0, 87163 +1, 87177 +2, 87177 +3, 87209 +4, 87189 +5, 87179 +6, 87181 +7, 87167 +0, 88409 +1, 88341 +2, 88371 +3, 88373 +4, 88387 +5, 88351 +6, 88361 +7, 88341 +0, 117865 +1, 117863 +2, 117903 +3, 117859 +4, 117915 +5, 117903 +6, 117863 +7, 117903 +0, 117865 +1, 117863 +2, 117903 +3, 117859 +4, 117915 +5, 117903 +6, 117863 +7, 117903 +0, 117865 +1, 117863 +2, 117903 +3, 117859 +4, 117915 +5, 117903 +6, 117863 +7, 117903 +0, 117865 +1, 117863 +2, 117903 +3, 117859 +4, 117915 +5, 117903 +6, 117863 +7, 117903 +0, 117865 +1, 117863 +2, 117903 +3, 117859 +4, 117915 +5, 117903 +6, 117863 +7, 117903 +0, 1709 +1, 1712 +2, 2032 +3, 117859 +4, 3312 +5, 1712 +6, 1712 +7, 1712 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..d8990633e2a7c608cf2964a9a3a24542f11d1ea5 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=256 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f2ef02b2ce407088f4f3b8b6bced62726aa24620 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log @@ -0,0 +1,26009 @@ +[15:00:18] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:00:18] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 15:00:20.046000 49721 torch/distributed/run.py:766] +W0826 15:00:20.046000 49721 torch/distributed/run.py:766] ***************************************** +W0826 15:00:20.046000 49721 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:00:20.046000 49721 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 76 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 4 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 15:00:38.149345567 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:00:38.149346145 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:00:38.150688724 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:00:38.150909635 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:00:38.151544873 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:00:38.151884365 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:00:38.153722652 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 15:00:38.339525530 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.692 seconds +time to initialize megatron (seconds): 7.232 +[after megatron is initialized] datetime: 2026-08-26 15:00:43 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8372566016 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.pre_mlp_layernorm.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.40.pre_mlp_layernorm.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.k_layernorm.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.pre_mlp_layernorm.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_qkv.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.q_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:00:48 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 15:00:48 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (4592.81, 4604.65) + train/valid/test-data-iterators-setup ..........: (75.11, 88.13) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 15:00:48 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32190 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32190 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=30952 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32614 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=30952 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32614 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=31373 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=31373 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32205 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32205 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=31723 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=31723 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=31105 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32137 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=31105 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32137 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=31814 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=31814 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32282 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32282 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=31919 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=31919 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32461 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32461 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=40960 expected_recv=30829 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=40960 expected_recv=30829 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31832 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31832 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31737 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31737 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=31362 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=31362 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=40960 expected_recv=31126 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=40960 expected_recv=31126 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32345 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32345 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=40960 expected_recv=31752 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32566 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=40960 expected_recv=31752 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32566 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32206 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32206 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=31033 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=31033 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=40960 expected_recv=31763 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=40960 expected_recv=31763 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=40960 expected_recv=30471 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=40960 expected_recv=31748 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=40960 expected_recv=30471 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=40960 expected_recv=31748 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=40960 expected_recv=31820 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=40960 expected_recv=31820 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=40960 expected_recv=31894 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=40960 expected_recv=31894 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=40960 expected_recv=31675 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=40960 expected_recv=31675 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=40960 expected_recv=30914 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31831 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=40960 expected_recv=30914 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31831 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31905 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31905 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=23 calls=1 budget=40960 expected_recv=31285 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=23 calls=1 budget=40960 expected_recv=31285 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=40960 expected_recv=31146 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=40960 expected_recv=31146 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32283 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32283 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=40960 expected_recv=31095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=40960 expected_recv=31095 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31625 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31625 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32179 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32179 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=25 calls=1 budget=40960 expected_recv=31593 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=25 calls=1 budget=40960 expected_recv=31593 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=40960 expected_recv=30473 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=40960 expected_recv=30473 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31943 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31943 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=26 calls=1 budget=40960 expected_recv=31852 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=26 calls=1 budget=40960 expected_recv=31852 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=26 calls=1 budget=40960 expected_recv=31716 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=26 calls=1 budget=40960 expected_recv=31716 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32363 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32363 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32194 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=29 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32194 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=29 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32365 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32365 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32280 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32280 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32359 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32359 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=32 calls=1 budget=40960 expected_recv=31570 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=32 calls=1 budget=40960 expected_recv=31570 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=33 calls=1 budget=40960 expected_recv=31387 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=33 calls=1 budget=40960 expected_recv=31387 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32182 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31623 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32182 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31623 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32367 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32367 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=28 calls=1 budget=40960 expected_recv=31847 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=28 calls=1 budget=40960 expected_recv=31847 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=36 calls=1 budget=40960 expected_recv=30806 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=37 calls=1 budget=40960 expected_recv=30890 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=36 calls=1 budget=40960 expected_recv=30806 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=37 calls=1 budget=40960 expected_recv=30890 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31945 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31945 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=30 calls=1 budget=40960 expected_recv=30602 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=30 calls=1 budget=40960 expected_recv=30602 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=38 calls=1 budget=40960 expected_recv=31975 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=38 calls=1 budget=40960 expected_recv=31975 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=31 calls=1 budget=40960 expected_recv=31522 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=31 calls=1 budget=40960 expected_recv=31522 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=39 calls=1 budget=40960 expected_recv=31295 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=39 calls=1 budget=40960 expected_recv=31295 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=38 calls=1 budget=40960 expected_recv=31595 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=38 calls=1 budget=40960 expected_recv=31595 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32113 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32113 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=40 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=34 calls=1 budget=40960 expected_recv=31039 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=40 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=34 calls=1 budget=40960 expected_recv=31039 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32498 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32367 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32498 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32367 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=43 calls=1 budget=40960 expected_recv=31714 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=43 calls=1 budget=40960 expected_recv=31714 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=43 calls=1 budget=40960 expected_recv=31854 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=43 calls=1 budget=40960 expected_recv=31854 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=44 calls=1 budget=40960 expected_recv=31877 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=44 calls=1 budget=40960 expected_recv=31877 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=46 calls=1 budget=40960 expected_recv=30723 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=46 calls=1 budget=40960 expected_recv=30723 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=45 calls=1 budget=40960 expected_recv=31680 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=45 calls=1 budget=40960 expected_recv=31680 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32401 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32401 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31814 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32584 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31814 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32584 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=41 calls=1 budget=40960 expected_recv=31422 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=41 calls=1 budget=40960 expected_recv=31422 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32184 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32184 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32128 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32128 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31755 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31755 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=48 total_expected_recv=1571071 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341760 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=48 total_expected_recv=1576257 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=337152 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=48 total_expected_recv=1569356 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344960 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=48 total_expected_recv=1574413 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=336768 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=48 total_expected_recv=1577256 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=335744 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=48 total_expected_recv=1574642 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=337792 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=48 total_expected_recv=1575856 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=339712 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=48 total_expected_recv=1564061 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=348544 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:01:18] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 29969.1 | throughput per GPU (TFLOP/s/GPU): 3.0 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095286E+01 | loss scale: 1.0 | grad norm: 54.363 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +Number of parameters in transformer block in billions: 29.90 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 72351.18896484375 | max allocated: 79924.4072265625 | reserved: 80338.0 | max reserved: 80338.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=31432 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=31432 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=31576 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=31576 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=30947 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=30947 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=31252 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32318 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=31252 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32318 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32322 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=31245 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32322 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=31245 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33022 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=31115 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33022 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=31115 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31783 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31784 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31783 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31784 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=30922 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=30922 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=30880 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=30880 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34816 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=31125 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=31125 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=40960 expected_recv=30714 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=40960 expected_recv=30714 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32302 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32302 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=40960 expected_recv=31265 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=40960 expected_recv=31265 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=40960 expected_recv=31415 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=40960 expected_recv=31415 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32030 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=40960 expected_recv=31845 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32030 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=40960 expected_recv=31845 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32314 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32314 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=40960 expected_recv=31542 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=40960 expected_recv=31542 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=40960 expected_recv=30652 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=40960 expected_recv=30652 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32267 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32267 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=40960 expected_recv=31301 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=40960 expected_recv=31301 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32265 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32265 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32191 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32209 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32191 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32316 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32209 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32316 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=40960 expected_recv=31732 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=40960 expected_recv=31732 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31954 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31954 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31614 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31614 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=40960 expected_recv=30924 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=40960 expected_recv=30924 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=40960 expected_recv=30607 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=40960 expected_recv=30607 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32961 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31695 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33074 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32961 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31695 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33074 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31896 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=23 calls=1 budget=40960 expected_recv=31668 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31896 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=23 calls=1 budget=40960 expected_recv=31668 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33040 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33040 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32260 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32260 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=24 calls=1 budget=40960 expected_recv=30783 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=24 calls=1 budget=40960 expected_recv=30783 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32084 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32084 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32244 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32244 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32047 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32047 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32333 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32333 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33043 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33043 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=27 calls=1 budget=40960 expected_recv=31527 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=26 calls=1 budget=40960 expected_recv=31521 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32119 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=27 calls=1 budget=40960 expected_recv=31527 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=26 calls=1 budget=40960 expected_recv=31521 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32119 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32102 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32102 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=28 calls=1 budget=40960 expected_recv=31788 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=28 calls=1 budget=40960 expected_recv=31788 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32284 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32284 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=29 calls=1 budget=40960 expected_recv=31940 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=29 calls=1 budget=40960 expected_recv=31940 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=30 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=30 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32259 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=31 calls=1 budget=40960 expected_recv=31685 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32259 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=31 calls=1 budget=40960 expected_recv=31685 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31945 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31945 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31621 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32178 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31621 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32178 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33071 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33071 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=33 calls=1 budget=40960 expected_recv=31415 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=33 calls=1 budget=40960 expected_recv=31415 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=34 calls=1 budget=40960 expected_recv=31009 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=34 calls=1 budget=40960 expected_recv=31009 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=35 calls=1 budget=40960 expected_recv=31930 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=35 calls=1 budget=40960 expected_recv=31930 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=35 calls=1 budget=40960 expected_recv=31638 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=35 calls=1 budget=40960 expected_recv=31638 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32009 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32009 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=36 calls=1 budget=40960 expected_recv=31560 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=36 calls=1 budget=40960 expected_recv=31560 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32059 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32059 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=38 calls=1 budget=40960 expected_recv=31696 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=38 calls=1 budget=40960 expected_recv=31696 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=37 calls=1 budget=40960 expected_recv=31508 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=37 calls=1 budget=40960 expected_recv=31508 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=38 calls=1 budget=40960 expected_recv=31872 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=38 calls=1 budget=40960 expected_recv=31872 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33050 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31913 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33050 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31913 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=39 calls=1 budget=40960 expected_recv=30516 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=39 calls=1 budget=40960 expected_recv=30516 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31655 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33016 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31655 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33016 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=41 calls=1 budget=40960 expected_recv=30551 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=42 calls=1 budget=40960 expected_recv=30473 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=41 calls=1 budget=40960 expected_recv=30551 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=42 calls=1 budget=40960 expected_recv=30473 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31663 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31663 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31906 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31906 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=44 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=44 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31850 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31718 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31850 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31718 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=46 calls=1 budget=40960 expected_recv=30605 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=46 calls=1 budget=40960 expected_recv=30605 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31947 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31947 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31621 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31621 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=48 total_expected_recv=1573344 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341376 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=48 total_expected_recv=1573610 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=338432 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=48 total_expected_recv=1571774 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=340736 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=48 total_expected_recv=1579862 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=335360 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=48 total_expected_recv=1568195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=48 total_expected_recv=1569671 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=48 total_expected_recv=1577775 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=335872 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=48 total_expected_recv=1568681 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:01:19] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 1250.1 | throughput per GPU (TFLOP/s/GPU): 71.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095473E+01 | loss scale: 1.0 | grad norm: 56.502 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32009 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=31559 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32009 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=31559 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=31527 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32040 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=31527 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32040 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=30803 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=30803 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=31760 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=31760 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32093 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32093 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=31007 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=31007 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32561 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32561 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=30471 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=30471 actual_valid_recv=30976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=30976 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=30761 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=30761 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32001 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32001 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=31781 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=31781 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34816 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=40960 expected_recv=30713 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=40960 expected_recv=30713 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=40960 expected_recv=30770 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=40960 expected_recv=30770 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=40960 expected_recv=31815 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=40960 expected_recv=31815 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=40960 expected_recv=30515 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=40960 expected_recv=30515 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=40960 expected_recv=31755 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=40960 expected_recv=31755 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33053 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33053 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=40960 expected_recv=31041 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=40960 expected_recv=30899 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=40960 expected_recv=31041 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=40960 expected_recv=30899 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=40960 expected_recv=30638 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32082 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=40960 expected_recv=30638 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32082 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32120 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32120 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32043 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32043 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32326 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32326 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31653 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31653 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=40960 expected_recv=31629 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31915 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=40960 expected_recv=31629 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31915 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32325 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32325 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32064 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32064 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=40960 expected_recv=31944 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=23 calls=1 budget=40960 expected_recv=31518 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=40960 expected_recv=31944 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=23 calls=1 budget=40960 expected_recv=31518 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=40960 expected_recv=30884 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=40960 expected_recv=30884 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32367 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=25 calls=1 budget=40960 expected_recv=31451 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32367 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=25 calls=1 budget=40960 expected_recv=31451 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=24 calls=1 budget=40960 expected_recv=31214 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=24 calls=1 budget=40960 expected_recv=31214 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=26 calls=1 budget=40960 expected_recv=30745 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33063 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=26 calls=1 budget=40960 expected_recv=30745 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33063 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=28 calls=1 budget=40960 expected_recv=30727 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=27 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=28 calls=1 budget=40960 expected_recv=30727 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=27 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=29 calls=1 budget=40960 expected_recv=30717 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=29 calls=1 budget=40960 expected_recv=30717 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32303 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32303 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=30 calls=1 budget=40960 expected_recv=31942 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=30 calls=1 budget=40960 expected_recv=31942 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=31 calls=1 budget=40960 expected_recv=31996 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32043 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=31 calls=1 budget=40960 expected_recv=31996 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32043 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=33 calls=1 budget=40960 expected_recv=30473 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=33 calls=1 budget=40960 expected_recv=30473 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=32 calls=1 budget=40960 expected_recv=30978 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=32 calls=1 budget=40960 expected_recv=30978 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=34 calls=1 budget=40960 expected_recv=31150 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=34 calls=1 budget=40960 expected_recv=31150 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=35 calls=1 budget=40960 expected_recv=31641 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32418 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=35 calls=1 budget=40960 expected_recv=31641 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32418 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32031 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32031 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31955 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31955 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=36 calls=1 budget=40960 expected_recv=31624 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=36 calls=1 budget=40960 expected_recv=31624 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32004 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32004 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33037 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33037 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31613 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32514 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31613 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=39 calls=1 budget=40960 expected_recv=31899 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32514 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=39 calls=1 budget=40960 expected_recv=31899 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=38 calls=1 budget=40960 expected_recv=31454 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=38 calls=1 budget=40960 expected_recv=31454 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=39 calls=1 budget=40960 expected_recv=31668 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=39 calls=1 budget=40960 expected_recv=31668 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33008 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=40 calls=1 budget=40960 expected_recv=31756 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=40 calls=1 budget=40960 expected_recv=31811 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33008 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=40 calls=1 budget=40960 expected_recv=31756 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=40 calls=1 budget=40960 expected_recv=31811 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=42 calls=1 budget=40960 expected_recv=31138 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31855 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31976 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=42 calls=1 budget=40960 expected_recv=31138 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31855 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31976 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=44 calls=1 budget=40960 expected_recv=31595 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=44 calls=1 budget=40960 expected_recv=31595 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33099 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33099 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31780 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31780 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32489 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32489 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32214 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=46 calls=1 budget=40960 expected_recv=31354 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32214 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=46 calls=1 budget=40960 expected_recv=31354 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32960 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32960 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=45 calls=1 budget=40960 expected_recv=31020 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=45 calls=1 budget=40960 expected_recv=31020 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32392 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32392 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=47 calls=1 budget=40960 expected_recv=31329 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=47 calls=1 budget=40960 expected_recv=31329 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33080 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33080 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=48 total_expected_recv=1570870 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=354688 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=48 total_expected_recv=1574804 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=353024 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=48 total_expected_recv=1566166 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=360704 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=48 total_expected_recv=1581859 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344192 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=48 total_expected_recv=1574781 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=353792 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=48 total_expected_recv=1570851 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=355328 peak_demand=33099 required_capacity=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=48 total_expected_recv=1572757 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=356608 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=48 total_expected_recv=1570824 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=356224 peak_demand=33098 required_capacity=1.010071 + [2026-08-26 15:01:21] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 1287.9 | throughput per GPU (TFLOP/s/GPU): 69.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008786E+01 | loss scale: 1.0 | grad norm: 180.776 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32557 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=31011 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32557 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=31011 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=31257 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=31257 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32274 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=31759 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32274 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=31759 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32259 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=31917 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32259 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=31917 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32000 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32000 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=31839 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=31839 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33092 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33092 actual_valid_recv=34944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34944 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=30769 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=30769 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=31144 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=31144 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=31643 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=31643 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32584 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32584 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32212 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32212 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=31886 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=31886 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32012 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32602 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32012 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32602 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=40960 expected_recv=31595 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=40960 expected_recv=31595 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=40960 expected_recv=31399 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=40960 expected_recv=31399 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=40960 expected_recv=31604 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=40960 expected_recv=31604 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33059 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33059 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32336 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32336 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32561 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32561 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32127 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32127 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32674 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32355 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32674 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32355 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32174 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33063 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32436 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32174 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33063 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32436 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32484 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32484 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32576 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32576 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=40960 expected_recv=31022 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=40960 expected_recv=31022 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=40960 expected_recv=31584 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=40960 expected_recv=31584 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32420 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=40960 expected_recv=31819 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=40960 expected_recv=31819 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32918 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32918 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32404 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32343 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32404 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32343 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=24 calls=1 budget=40960 expected_recv=31117 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=24 calls=1 budget=40960 expected_recv=31117 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33001 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33039 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33020 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33001 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33039 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32122 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33020 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32122 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32374 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32374 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=26 calls=1 budget=40960 expected_recv=31868 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=26 calls=1 budget=40960 expected_recv=31868 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32399 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32399 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32637 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32637 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32497 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32497 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=28 calls=1 budget=40960 expected_recv=31947 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32391 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=28 calls=1 budget=40960 expected_recv=31947 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32391 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=30 calls=1 budget=40960 expected_recv=31935 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=30 calls=1 budget=40960 expected_recv=31935 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32261 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32261 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32600 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32600 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33088 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32948 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33088 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32948 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=31 calls=1 budget=40960 expected_recv=31892 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=31 calls=1 budget=40960 expected_recv=31892 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32408 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32408 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32155 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32155 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=33 calls=1 budget=40960 expected_recv=31863 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=33 calls=1 budget=40960 expected_recv=31863 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32239 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32239 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32434 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32434 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32457 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32457 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32280 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32280 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32104 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32104 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32454 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32599 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32454 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32599 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32051 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32051 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32245 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32245 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=39 calls=1 budget=40960 expected_recv=31627 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=39 calls=1 budget=40960 expected_recv=31627 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32423 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32423 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32154 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32154 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=42 calls=1 budget=40960 expected_recv=31397 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=42 calls=1 budget=40960 expected_recv=31397 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33024 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=43 calls=1 budget=40960 expected_recv=31237 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=43 calls=1 budget=40960 expected_recv=31237 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32582 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32508 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32582 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32425 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32508 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32425 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32396 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32513 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32396 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32513 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32520 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32520 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33029 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=45 calls=1 budget=40960 expected_recv=31982 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=46 calls=1 budget=40960 expected_recv=31490 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33029 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=45 calls=1 budget=40960 expected_recv=31982 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=46 calls=1 budget=40960 expected_recv=31490 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=47 calls=1 budget=40960 expected_recv=31701 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33011 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=47 calls=1 budget=40960 expected_recv=31701 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33011 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=48 total_expected_recv=1574944 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=365184 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=48 total_expected_recv=1572404 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368512 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=48 total_expected_recv=1574820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364800 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=48 total_expected_recv=1571340 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=367872 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=48 total_expected_recv=1569201 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=371328 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=48 total_expected_recv=1571289 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=366336 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=48 total_expected_recv=1574339 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=367616 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=48 total_expected_recv=1574575 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364416 peak_demand=33098 required_capacity=1.010071 + [2026-08-26 15:01:21] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 719.4 | throughput per GPU (TFLOP/s/GPU): 123.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604043E+00 | loss scale: 1.0 | grad norm: 8.752 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=31141 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=31141 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32233 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32233 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=31335 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=31335 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=31994 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=31994 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=31575 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=31575 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32073 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32073 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=31960 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32215 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=31960 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32215 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32293 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32293 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=31778 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=31778 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=31730 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=31730 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33054 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33054 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=31839 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=31839 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32455 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32330 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32455 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32330 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32277 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32277 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33059 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33059 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32260 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32223 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32260 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32223 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32526 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32526 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32321 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32321 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=40960 expected_recv=31709 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=40960 expected_recv=31709 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32400 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32375 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32400 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32353 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32375 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32353 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32403 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32403 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32488 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32488 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32084 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32084 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32396 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32396 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32405 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32265 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32405 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32265 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32307 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32307 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32304 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32304 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32332 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32485 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32332 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32485 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32296 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32296 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32485 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32485 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32434 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32434 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32473 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32473 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32517 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32517 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32098 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32098 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32471 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32471 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32492 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32397 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32395 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32492 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32397 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32395 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32413 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32413 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32304 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32304 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32321 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32321 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32342 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32355 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32342 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32355 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32282 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32282 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=36 calls=1 budget=40960 expected_recv=31850 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=36 calls=1 budget=40960 expected_recv=31850 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32362 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32362 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32412 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32375 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32412 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32375 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32471 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32471 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32459 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32459 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32396 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32396 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32317 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32317 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32262 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32334 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32262 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32334 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32097 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32097 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32390 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32390 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32424 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32320 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32381 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32424 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32320 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32381 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32484 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32484 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=45 calls=1 budget=40960 expected_recv=31452 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=45 calls=1 budget=40960 expected_recv=31452 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32077 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32077 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=48 total_expected_recv=1574855 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=361856 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=48 total_expected_recv=1572739 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364416 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=48 total_expected_recv=1572902 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=365696 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=48 total_expected_recv=1573384 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=366848 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=48 total_expected_recv=1569156 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=370688 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=48 total_expected_recv=1571237 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368640 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=48 total_expected_recv=1574485 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364928 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=48 total_expected_recv=1574154 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364672 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:01:22] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 715.5 | throughput per GPU (TFLOP/s/GPU): 124.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307577E+00 | loss scale: 1.0 | grad norm: 9.631 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32348 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=31220 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32348 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=31220 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=31345 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=31345 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=31278 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32290 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=31278 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32290 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=31687 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=31687 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32195 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32195 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=30789 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=30789 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33093 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33093 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32302 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32302 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=31556 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=31556 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32932 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32932 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=31618 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=31618 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32207 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32207 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=31688 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=40960 expected_recv=31627 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=31688 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=40960 expected_recv=31627 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32320 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32320 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=40960 expected_recv=31282 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=40960 expected_recv=31282 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32180 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32420 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32180 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32136 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32136 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32316 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32316 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33075 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33075 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32127 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32127 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32390 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32390 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32450 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32450 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32450 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32450 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32133 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32133 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32269 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32269 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32454 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32454 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32479 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32479 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32977 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32503 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32503 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32418 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32418 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32566 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32566 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32492 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32468 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32492 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32468 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32450 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=27 calls=1 budget=40960 expected_recv=31782 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32450 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=27 calls=1 budget=40960 expected_recv=31782 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32189 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32189 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32484 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32484 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32402 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32402 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=31 calls=1 budget=40960 expected_recv=31919 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=31 calls=1 budget=40960 expected_recv=31919 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32420 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33025 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33025 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32382 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32415 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32382 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32415 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32183 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32183 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32252 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32252 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32409 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32409 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32030 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32030 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32099 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32099 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=39 calls=1 budget=40960 expected_recv=31801 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=39 calls=1 budget=40960 expected_recv=31801 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32394 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32394 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32152 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32152 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32452 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32452 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32407 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32407 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32416 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32415 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32416 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32415 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32413 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32413 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32492 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32492 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=48 total_expected_recv=1571361 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378112 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=48 total_expected_recv=1572291 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375296 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=48 total_expected_recv=1574204 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373888 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=48 total_expected_recv=1570070 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378624 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=48 total_expected_recv=1573655 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375936 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=48 total_expected_recv=1572098 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375424 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=48 total_expected_recv=1575595 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372608 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=48 total_expected_recv=1573638 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375168 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:01:23] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 737.8 | throughput per GPU (TFLOP/s/GPU): 120.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.118724E+00 | loss scale: 1.0 | grad norm: 2.924 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=31907 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32583 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32174 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=31907 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32583 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32174 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=31866 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32109 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=31866 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32109 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=31329 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32238 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=31329 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32238 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=31660 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33046 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32981 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=31660 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33046 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32981 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=31040 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=31040 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=31080 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=31080 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32301 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32301 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32531 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32531 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32041 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32041 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32395 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32152 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32395 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32152 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=40960 expected_recv=31506 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=40960 expected_recv=31506 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32161 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32161 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=40960 expected_recv=31569 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=40960 expected_recv=31569 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32454 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32454 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32085 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32085 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32429 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32429 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=40960 expected_recv=31702 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=40960 expected_recv=31702 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32425 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32425 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32238 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32238 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=40960 expected_recv=31812 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=40960 expected_recv=31812 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=40960 expected_recv=31129 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=23 calls=1 budget=40960 expected_recv=31523 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=40960 expected_recv=31129 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=23 calls=1 budget=40960 expected_recv=31523 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32124 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32124 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=25 calls=1 budget=40960 expected_recv=31931 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=25 calls=1 budget=40960 expected_recv=31931 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33020 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33020 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32539 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32539 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32366 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32392 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32366 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32392 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32455 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32455 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32462 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32462 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=27 calls=1 budget=40960 expected_recv=31796 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=27 calls=1 budget=40960 expected_recv=31796 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32375 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32375 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32424 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32424 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=29 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32397 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=29 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32397 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=30 calls=1 budget=40960 expected_recv=31512 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=30 calls=1 budget=40960 expected_recv=31512 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32416 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=32 calls=1 budget=40960 expected_recv=31875 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32416 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=32 calls=1 budget=40960 expected_recv=31875 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32348 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32348 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=34 calls=1 budget=40960 expected_recv=31760 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=34 calls=1 budget=40960 expected_recv=31760 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32255 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32255 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32412 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32412 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=38 calls=1 budget=40960 expected_recv=31788 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=38 calls=1 budget=40960 expected_recv=31788 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32117 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32117 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=42 calls=1 budget=40960 expected_recv=31414 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=42 calls=1 budget=40960 expected_recv=31414 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32458 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32458 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32410 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32410 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=43 calls=1 budget=40960 expected_recv=31911 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=43 calls=1 budget=40960 expected_recv=31911 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32462 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32462 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32402 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32402 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=48 total_expected_recv=1573668 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374016 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=48 total_expected_recv=1570049 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378496 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=48 total_expected_recv=1575100 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373248 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=48 total_expected_recv=1571037 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378624 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=48 total_expected_recv=1574309 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374400 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=48 total_expected_recv=1574378 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374272 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=48 total_expected_recv=1569070 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377600 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=48 total_expected_recv=1575301 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374144 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:01:24] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 735.9 | throughput per GPU (TFLOP/s/GPU): 121.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.007059E+00 | loss scale: 1.0 | grad norm: 3.702 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=30930 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=30930 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=30783 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=30783 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32348 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32348 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=31554 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=31554 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=31761 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=31761 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=31534 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=31534 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32263 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32344 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32263 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32344 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32228 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32228 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32389 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32389 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=31749 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32145 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=31749 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32145 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32295 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32295 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32583 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32583 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=31681 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=31681 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=40960 expected_recv=31522 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=40960 expected_recv=31522 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32457 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32457 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32118 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32118 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32078 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32078 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32531 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32531 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32355 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32355 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32602 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32602 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32326 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32326 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32388 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32165 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32388 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32165 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32393 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32393 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32313 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32313 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32111 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=40960 expected_recv=31786 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32111 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=40960 expected_recv=31786 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=40960 expected_recv=31665 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=40960 expected_recv=31665 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32559 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32559 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=40960 expected_recv=31776 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=40960 expected_recv=31776 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32168 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32168 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=27 calls=1 budget=40960 expected_recv=31702 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=27 calls=1 budget=40960 expected_recv=31702 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32373 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32373 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32227 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32227 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32530 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32339 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32339 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32483 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32483 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=37 calls=1 budget=40960 expected_recv=31791 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=37 calls=1 budget=40960 expected_recv=31791 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32399 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32399 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32117 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32117 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32393 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32393 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=40 calls=1 budget=40960 expected_recv=31910 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=40 calls=1 budget=40960 expected_recv=31910 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=42 calls=1 budget=40960 expected_recv=31784 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=42 calls=1 budget=40960 expected_recv=31784 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32067 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32111 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32067 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32111 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32436 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32436 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=47 calls=1 budget=40960 expected_recv=31995 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=47 calls=1 budget=40960 expected_recv=31995 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32473 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32473 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=48 total_expected_recv=1575478 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375680 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=48 total_expected_recv=1566961 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=384640 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=48 total_expected_recv=1573222 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378368 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=48 total_expected_recv=1573757 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379008 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=48 total_expected_recv=1575025 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376192 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=48 total_expected_recv=1575019 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377344 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=48 total_expected_recv=1574815 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378112 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=48 total_expected_recv=1568635 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=383104 peak_demand=33098 required_capacity=1.010071 + [2026-08-26 15:01:25] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 1091.5 | throughput per GPU (TFLOP/s/GPU): 81.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.861445E+00 | loss scale: 1.0 | grad norm: 3.114 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=30944 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=30944 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=31612 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=31956 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=31612 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=31956 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=30992 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=30992 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=31851 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=31851 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32227 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32227 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=31004 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=31004 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=31057 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=31057 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=30861 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33026 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=31876 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=30861 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33026 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=31876 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32071 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32071 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32003 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32003 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33043 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33017 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33043 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33017 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32105 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32105 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32544 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32542 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32371 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32544 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32542 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32371 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32228 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32281 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32228 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32281 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32516 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32516 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=40960 expected_recv=31460 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=40960 expected_recv=31460 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32330 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32330 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32425 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32337 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32224 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32425 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32337 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32224 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32343 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32343 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32958 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33018 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32958 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33018 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32240 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32240 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32555 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32388 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32555 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32388 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=40960 expected_recv=31449 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=40960 expected_recv=31449 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32272 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32272 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32318 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32318 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=40960 expected_recv=31981 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=40960 expected_recv=31981 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32312 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32312 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32544 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32544 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32121 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32121 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32377 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=24 calls=1 budget=40960 expected_recv=31780 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32377 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=24 calls=1 budget=40960 expected_recv=31780 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32401 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=26 calls=1 budget=40960 expected_recv=31768 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32401 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=26 calls=1 budget=40960 expected_recv=31768 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32119 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=28 calls=1 budget=40960 expected_recv=31803 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32119 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=28 calls=1 budget=40960 expected_recv=31803 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32104 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32104 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32143 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32143 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32109 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32109 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32154 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32154 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32599 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32599 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=35 calls=1 budget=40960 expected_recv=31791 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=35 calls=1 budget=40960 expected_recv=31791 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32306 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32306 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32401 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32401 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32302 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32302 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=40 calls=1 budget=40960 expected_recv=31730 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32024 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=40 calls=1 budget=40960 expected_recv=31730 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32024 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=42 calls=1 budget=40960 expected_recv=31780 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=42 calls=1 budget=40960 expected_recv=31780 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32227 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32227 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32115 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32115 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=47 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=46 calls=1 budget=40960 expected_recv=31906 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=47 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=46 calls=1 budget=40960 expected_recv=31906 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=48 total_expected_recv=1573105 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376960 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=48 total_expected_recv=1571483 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377216 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=48 total_expected_recv=1570227 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379264 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=48 total_expected_recv=1570252 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380544 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=48 total_expected_recv=1574195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375936 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=48 total_expected_recv=1576891 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372864 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=48 total_expected_recv=1572392 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378240 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=48 total_expected_recv=1574367 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378368 peak_demand=33098 required_capacity=1.010071 + [2026-08-26 15:01:25] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 722.0 | throughput per GPU (TFLOP/s/GPU): 123.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.820069E+00 | loss scale: 1.0 | grad norm: 2.904 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32072 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33053 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=31538 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32072 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33053 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=31538 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=30948 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=30948 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32041 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32134 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32041 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32134 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=31005 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=31005 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32561 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32561 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=31289 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=31289 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32279 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32279 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=31308 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=31308 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32548 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32548 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=31833 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=31833 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=31026 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=31026 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=40960 expected_recv=31844 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=40960 expected_recv=31844 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32057 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32057 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32004 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=40960 expected_recv=31914 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32004 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=40960 expected_recv=31914 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32331 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32331 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32408 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32408 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32254 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32254 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=40960 expected_recv=31237 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=40960 expected_recv=31237 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=40960 expected_recv=31130 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=40960 expected_recv=31130 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=40960 expected_recv=31615 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=40960 expected_recv=31615 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32354 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32354 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32131 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32131 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=40960 expected_recv=31735 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=40960 expected_recv=31735 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=40960 expected_recv=31908 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=40960 expected_recv=31908 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=40960 expected_recv=31439 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=40960 expected_recv=31439 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=40960 expected_recv=31463 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=40960 expected_recv=31463 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=23 calls=1 budget=40960 expected_recv=31881 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=23 calls=1 budget=40960 expected_recv=31881 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32361 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32361 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32340 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32340 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32225 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32225 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=25 calls=1 budget=40960 expected_recv=31783 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=25 calls=1 budget=40960 expected_recv=31783 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=26 calls=1 budget=40960 expected_recv=31834 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32910 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=26 calls=1 budget=40960 expected_recv=31834 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32910 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32371 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32371 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32374 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32374 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32368 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=29 calls=1 budget=40960 expected_recv=31757 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32368 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=29 calls=1 budget=40960 expected_recv=31757 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32429 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32429 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32350 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32350 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32410 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32410 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=33 calls=1 budget=40960 expected_recv=31456 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=33 calls=1 budget=40960 expected_recv=31456 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32897 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32897 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32436 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32436 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32434 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32434 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32141 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32141 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32154 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32154 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32071 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32071 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=43 calls=1 budget=40960 expected_recv=31453 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=42 calls=1 budget=40960 expected_recv=31455 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=43 calls=1 budget=40960 expected_recv=31453 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=42 calls=1 budget=40960 expected_recv=31455 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32061 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32061 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32052 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32052 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=47 calls=1 budget=40960 expected_recv=31841 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=47 calls=1 budget=40960 expected_recv=31841 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=48 total_expected_recv=1573034 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376192 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=48 total_expected_recv=1574327 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374400 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=48 total_expected_recv=1569897 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379264 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=48 total_expected_recv=1574031 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375296 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=48 total_expected_recv=1568466 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380928 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=48 total_expected_recv=1574590 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375680 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=48 total_expected_recv=1576372 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372096 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=48 total_expected_recv=1572195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379136 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:01:26] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 730.4 | throughput per GPU (TFLOP/s/GPU): 122.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.781345E+00 | loss scale: 1.0 | grad norm: 2.914 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32264 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=31407 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32264 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=31407 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=31151 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=31151 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32415 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32415 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31752 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31752 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31815 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31815 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32242 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=31326 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32242 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=31326 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=31532 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=31532 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32038 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=30765 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32038 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=30765 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=30569 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=30569 actual_valid_recv=30848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=31433 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=30848 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32999 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=31433 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32377 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32377 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=30962 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=30962 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=40960 expected_recv=31988 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=40960 expected_recv=31988 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32158 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32158 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=40960 expected_recv=31769 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=40960 expected_recv=31769 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32350 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32350 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32262 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32262 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=40960 expected_recv=31635 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=40960 expected_recv=31635 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=40960 expected_recv=31659 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=40960 expected_recv=31659 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=40960 expected_recv=31259 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=40960 expected_recv=31259 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=40960 expected_recv=31769 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=40960 expected_recv=31769 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=40960 expected_recv=31714 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=40960 expected_recv=31714 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=40960 expected_recv=31439 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=40960 expected_recv=31439 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=40960 expected_recv=31998 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=40960 expected_recv=31998 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33068 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33068 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=40960 expected_recv=30802 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=40960 expected_recv=30802 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=40960 expected_recv=31671 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=40960 expected_recv=31671 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32338 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32338 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32546 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32546 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32129 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32129 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32373 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32373 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32376 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32376 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=24 calls=1 budget=40960 expected_recv=31314 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=24 calls=1 budget=40960 expected_recv=31314 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=26 calls=1 budget=40960 expected_recv=31597 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=26 calls=1 budget=40960 expected_recv=31597 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32131 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=28 calls=1 budget=40960 expected_recv=31372 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32131 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=28 calls=1 budget=40960 expected_recv=31372 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=30 calls=1 budget=40960 expected_recv=31792 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=30 calls=1 budget=40960 expected_recv=31792 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32123 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32123 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32107 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32602 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32107 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32602 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=34 calls=1 budget=40960 expected_recv=31733 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=34 calls=1 budget=40960 expected_recv=31733 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=33 calls=1 budget=40960 expected_recv=31157 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=33 calls=1 budget=40960 expected_recv=31157 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33045 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=35 calls=1 budget=40960 expected_recv=31735 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32155 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=35 calls=1 budget=40960 expected_recv=31735 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32155 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32474 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32474 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=38 calls=1 budget=40960 expected_recv=31520 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=38 calls=1 budget=40960 expected_recv=31520 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=39 calls=1 budget=40960 expected_recv=31748 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=39 calls=1 budget=40960 expected_recv=31748 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=40 calls=1 budget=40960 expected_recv=31293 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=40 calls=1 budget=40960 expected_recv=31293 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=41 calls=1 budget=40960 expected_recv=31805 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=41 calls=1 budget=40960 expected_recv=31805 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=42 calls=1 budget=40960 expected_recv=31816 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=43 calls=1 budget=40960 expected_recv=31439 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=42 calls=1 budget=40960 expected_recv=31816 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=43 calls=1 budget=40960 expected_recv=31439 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=46 calls=1 budget=40960 expected_recv=31784 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=46 calls=1 budget=40960 expected_recv=31784 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=45 calls=1 budget=40960 expected_recv=31817 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=45 calls=1 budget=40960 expected_recv=31817 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=48 total_expected_recv=1571791 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376448 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=48 total_expected_recv=1571972 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376832 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=48 total_expected_recv=1569790 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377600 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=48 total_expected_recv=1575441 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372352 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=48 total_expected_recv=1580067 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368000 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=48 total_expected_recv=1567189 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380672 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=48 total_expected_recv=1571442 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376320 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=48 total_expected_recv=1575220 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374016 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:01:27] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 724.3 | throughput per GPU (TFLOP/s/GPU): 123.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.773178E+00 | loss scale: 1.0 | grad norm: 2.826 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32346 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32112 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32206 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32346 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32112 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32206 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=31142 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33027 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=31142 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33027 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=31119 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=31119 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=30928 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=30928 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=30775 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=30775 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=30778 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=30778 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=31762 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32184 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=31762 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32184 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33077 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33077 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=31083 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=31083 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=30806 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32998 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=30806 actual_valid_recv=31104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=31104 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32998 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32399 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32399 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=31503 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=31503 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=40960 expected_recv=31774 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=40960 expected_recv=31774 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32190 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32190 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=40960 expected_recv=31881 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32646 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=40960 expected_recv=31881 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32646 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32137 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32137 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32275 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32275 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32323 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=40960 expected_recv=31272 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32323 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=40960 expected_recv=31272 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=40960 expected_recv=31116 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=40960 expected_recv=31116 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=40960 expected_recv=31570 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=40960 expected_recv=31570 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=40960 expected_recv=31341 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=40960 expected_recv=31341 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32027 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=40960 expected_recv=31446 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32027 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=40960 expected_recv=31446 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32485 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32485 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=40960 expected_recv=31589 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=40960 expected_recv=31589 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=40960 expected_recv=31693 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=40960 expected_recv=31693 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=40960 expected_recv=31962 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=40960 expected_recv=31962 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32240 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32240 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32407 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32001 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32407 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32001 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32579 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32579 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33078 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32110 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33078 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32110 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32365 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32365 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=26 calls=1 budget=40960 expected_recv=31246 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=26 calls=1 budget=40960 expected_recv=31246 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=28 calls=1 budget=40960 expected_recv=31972 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=27 calls=1 budget=40960 expected_recv=31958 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=28 calls=1 budget=40960 expected_recv=31972 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=27 calls=1 budget=40960 expected_recv=31958 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33040 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32267 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33040 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32267 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=30 calls=1 budget=40960 expected_recv=31461 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=30 calls=1 budget=40960 expected_recv=31461 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=29 calls=1 budget=40960 expected_recv=31853 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=29 calls=1 budget=40960 expected_recv=31853 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32197 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32197 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32489 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32489 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32360 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32360 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32172 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32172 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32082 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32082 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32387 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32387 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32103 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32103 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32423 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32423 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32141 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32141 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=40 calls=1 budget=40960 expected_recv=31721 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=40 calls=1 budget=40960 expected_recv=31721 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=41 calls=1 budget=40960 expected_recv=31894 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=41 calls=1 budget=40960 expected_recv=31894 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32974 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=42 calls=1 budget=40960 expected_recv=31716 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=43 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32974 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=42 calls=1 budget=40960 expected_recv=31716 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=43 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32455 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32174 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32455 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32174 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32111 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32111 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32115 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32115 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=46 calls=1 budget=40960 expected_recv=31752 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=46 calls=1 budget=40960 expected_recv=31752 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32398 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32398 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=48 total_expected_recv=1573727 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373248 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=48 total_expected_recv=1574569 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374016 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=48 total_expected_recv=1571606 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376064 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=48 total_expected_recv=1568576 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379392 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=48 total_expected_recv=1572352 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375936 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=48 total_expected_recv=1575436 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373120 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=48 total_expected_recv=1574252 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372864 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=48 total_expected_recv=1572394 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377088 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:01:28] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 722.7 | throughput per GPU (TFLOP/s/GPU): 123.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.744781E+00 | loss scale: 1.0 | grad norm: 2.735 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +[after training is done] datetime: 2026-08-26 15:01:28 +W0826 15:01:28.747000 49721 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:01:28.748000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49814 closing signal SIGTERM +W0826 15:01:28.748000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49815 closing signal SIGTERM +W0826 15:01:28.749000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49816 closing signal SIGTERM +W0826 15:01:28.749000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49817 closing signal SIGTERM +W0826 15:01:28.751000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49818 closing signal SIGTERM +W0826 15:01:28.752000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49819 closing signal SIGTERM +W0826 15:01:28.753000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49820 closing signal SIGTERM +W0826 15:01:28.753000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49821 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 49721 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..a9a0a1fb71acd0728768139446cac6c7565fd156 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -0,0 +1,8 @@ +model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir +QWEN3_235B ultraep measure 1.25 OK 122751 1 0.0 1.01004 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +QWEN3_235B ultraep validate 1.02 ASSERT 54649 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1 +QWEN3_30B ultraep measure 1.25 OK 117915 1 0.0 1.010101 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +QWEN3_30B ultraep validate 1.02 ASSERT 48657 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1 +DSV2_LITE_L24 ultraep measure 1.25 OK 130023 1 0.0 1.010071 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +DSV2_LITE_L24 ultraep validate 1.02 ASSERT 39855 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1 +QWEN2_57B ultraep measure 1.25 FAIL 47627 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c203cb0e891ef086e0affffdee5e029192210ff9 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -0,0 +1,5 @@ +model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir +QWEN3_235B ultraep ERROR_ASSERT +QWEN3_30B ultraep ERROR_ASSERT +DSV2_LITE_L24 ultraep ERROR_ASSERT +QWEN2_57B ultraep ERROR_FAIL diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7df05f5558f3b7aecfa9f1687654d71d266c4a29 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -0,0 +1,233 @@ +[ + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "DSV2_LITE_L24", + "cuda_graph": "0", + "dense_first_layers": "1", + "drop_expected_assignments": 108527616.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 276, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "23", + "num_experts": "64", + "num_layers": "24", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 49647.0, + "peak_demand_mean": 49645.270833333336, + "peak_demand_min": 49644.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.02649873914445656, + "post_physical_rank_cv_mean": 0.020751556297178456, + "post_physical_rank_cv_min": 0.010393449619293449, + "post_physical_rank_cv_n": 276, + "post_physical_rank_imax_max": 1.01007080078125, + "post_physical_rank_imax_mean": 1.0100281941142057, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 276, + "pre_home_rank_cv_max": 1.3897787536248405, + "pre_home_rank_cv_mean": 0.5881957299328056, + "pre_home_rank_cv_min": 0.06749209466238901, + "pre_home_rank_cv_n": 276, + "pre_home_rank_imax_max": 4.4163818359375, + "pre_home_rank_imax_mean": 2.107403446510794, + "pre_home_rank_imax_min": 1.1109822591145833, + "pre_home_rank_imax_n": 276, + "records": { + "drop_dispatch": 2208, + "drop_experts": 2208, + "drop_setup": 2208, + "ultraep_drop": 96, + "ultraep_post": 2208, + "ultraep_pre": 2208 + }, + "required_capacity_max": 1.010071, + "required_capacity_mean": 1.0100354270833332, + "required_capacity_min": 1.01001, + "required_capacity_n": 96, + "router_balancing": "seq_aux_loss", + "run_dir": "DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1", + "seq_length": "8192", + "shared_expert_overlap": "0", + "topk": "6", + "valid_max_rank_reduction_max": 0.7712991882952357, + "valid_max_rank_reduction_mean": 0.46558143481151487, + "valid_max_rank_reduction_min": 0.09086747120332558, + "valid_max_rank_reduction_n": 276, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "QWEN3_235B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 75497472.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "16", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 144, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "12", + "num_experts": "128", + "num_layers": "12", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 66194.0, + "peak_demand_mean": 66192.60416666667, + "peak_demand_min": 66192.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.026489118276835043, + "post_physical_rank_cv_mean": 0.014698204478704438, + "post_physical_rank_cv_min": 0.004171709396704442, + "post_physical_rank_cv_n": 144, + "post_physical_rank_imax_max": 1.010040283203125, + "post_physical_rank_imax_mean": 1.0100169711642795, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 144, + "pre_home_rank_cv_max": 1.3850730653619154, + "pre_home_rank_cv_mean": 0.8051429963522313, + "pre_home_rank_cv_min": 0.3167748701963148, + "pre_home_rank_cv_n": 144, + "pre_home_rank_imax_max": 4.0391845703125, + "pre_home_rank_imax_mean": 2.465598636203342, + "pre_home_rank_imax_min": 1.2317352294921875, + "pre_home_rank_imax_n": 144, + "records": { + "drop_dispatch": 1152, + "drop_experts": 1152, + "drop_setup": 1152, + "ultraep_drop": 96, + "ultraep_post": 1152, + "ultraep_pre": 1152 + }, + "required_capacity_max": 1.01004, + "required_capacity_mean": 1.0100190625, + "required_capacity_min": 1.01001, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1", + "seq_length": "8192", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7499471123334038, + "valid_max_rank_reduction_mean": 0.5612683072031022, + "valid_max_rank_reduction_min": 0.17998587763091065, + "valid_max_rank_reduction_n": 144, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "QWEN3_30B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 150994944.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "16", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 576, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "48", + "num_experts": "128", + "num_layers": "48", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33099.0, + "peak_demand_mean": 33097.229166666664, + "peak_demand_min": 33096.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.02649488696921753, + "post_physical_rank_cv_mean": 0.013615268745845822, + "post_physical_rank_cv_min": 0.0044932923824075426, + "post_physical_rank_cv_n": 576, + "post_physical_rank_imax_max": 1.010101318359375, + "post_physical_rank_imax_mean": 1.0100238058302138, + "post_physical_rank_imax_min": 1.00994873046875, + "post_physical_rank_imax_n": 576, + "pre_home_rank_cv_max": 1.577101326466163, + "pre_home_rank_cv_mean": 0.7366559250184537, + "pre_home_rank_cv_min": 0.14094636542440406, + "pre_home_rank_cv_n": 576, + "pre_home_rank_imax_max": 4.9957275390625, + "pre_home_rank_imax_mean": 2.2919526629977756, + "pre_home_rank_imax_min": 1.2030029296875, + "pre_home_rank_imax_n": 576, + "records": { + "drop_dispatch": 4608, + "drop_experts": 4608, + "drop_setup": 4608, + "ultraep_drop": 96, + "ultraep_post": 4608, + "ultraep_pre": 4608 + }, + "required_capacity_max": 1.010101, + "required_capacity_mean": 1.0100472291666667, + "required_capacity_min": 1.01001, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7978252901649359, + "valid_max_rank_reduction_mean": 0.5268142670026216, + "valid_max_rank_reduction_min": 0.1604261796042618, + "valid_max_rank_reduction_n": 576, + "zero_drop": 1 + } +] \ No newline at end of file diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7a90f8017942c8e50cda3fc5e415727980e08049 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -0,0 +1,4 @@ +run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches drop_expected_assignments drop_rate_pct_max drop_rate_pct_mean drop_rate_pct_min drop_rate_pct_n drop_rate_pct_overall dropped_tokens_sum_all_records duplicate_vector_mismatches global_snapshots incomplete_snapshots malformed_records observed_dropped_tokens peak_demand_max peak_demand_mean peak_demand_min peak_demand_n peak_rank_overflow_max peak_rank_overflow_mean peak_rank_overflow_min peak_rank_overflow_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop +DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 natural DSV2_LITE_L24 ultraep 64 6 8 24 23 1 8192 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-3 0 0 108527616.0 0.0 0.0 0.0 96 0.0 0.0 0 276 0 0 0.0 49647.0 49645.270833333336 49644.0 96 0.0 0.0 0.0 96 0.02649873914445656 0.020751556297178456 0.010393449619293449 276 1.01007080078125 1.0100281941142057 1.010009765625 276 1.3897787536248405 0.5881957299328056 0.06749209466238901 276 4.4163818359375 2.107403446510794 1.1109822591145833 276 1.010071 1.0100354270833332 1.01001 96 0.7712991882952357 0.46558143481151487 0.09086747120332558 276 1 +QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 natural QWEN3_235B ultraep 128 8 16 12 12 0 8192 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 75497472.0 0.0 0.0 0.0 96 0.0 0.0 0 144 0 0 0.0 66194.0 66192.60416666667 66192.0 96 0.0 0.0 0.0 96 0.026489118276835043 0.014698204478704438 0.004171709396704442 144 1.010040283203125 1.0100169711642795 1.010009765625 144 1.3850730653619154 0.8051429963522313 0.3167748701963148 144 4.0391845703125 2.465598636203342 1.2317352294921875 144 1.01004 1.0100190625 1.01001 96 0.7499471123334038 0.5612683072031022 0.17998587763091065 144 1 +QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 natural QWEN3_30B ultraep 128 8 16 48 48 0 4096 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 150994944.0 0.0 0.0 0.0 96 0.0 0.0 0 576 0 0 0.0 33099.0 33097.229166666664 33096.0 96 0.0 0.0 0.0 96 0.02649488696921753 0.013615268745845822 0.0044932923824075426 576 1.010101318359375 1.0100238058302138 1.00994873046875 576 1.577101326466163 0.7366559250184537 0.14094636542440406 576 4.9957275390625 2.2919526629977756 1.2030029296875 576 1.010101 1.0100472291666667 1.01001 96 0.7978252901649359 0.5268142670026216 0.1604261796042618 576 1 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b43c1f2b0e9b0ac4fcd2b83c5f1b476438fef49b --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -0,0 +1,8 @@ +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +QWEN3_235B ultraep matched 16 8192 8 1.25 none 1 OK 339.56 338.10 122751 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +QWEN3_235B ultraep matched 16 8192 8 1.02 none 1 ASSERT NA NA 54649 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1 +QWEN3_30B ultraep matched 16 4096 8 1.25 none 1 OK 780.66 722.00 117915 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +QWEN3_30B ultraep matched 16 4096 8 1.02 none 1 ASSERT NA NA 48657 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1 +DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none 1 OK 641.99 639.30 130023 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +DSV2_LITE_L24 ultraep matched 8 8192 8 1.02 none 1 ASSERT NA NA 39855 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1 +QWEN2_57B ultraep matched 8 4096 8 1.25 none 1 FAIL NA NA 47627 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1